Platform specific largest floating point type

CONTRIBUTING.md

@@ -9,7 +9,9 @@ linting, testing, and building the documentation, run the following:
 git clone https://github.com/pytorch/torcheval
 cd torcheval
 pip install -r requirements.txt
+pip install -r audio-requirements.txt
 pip install -r dev-requirements.txt
+pip install -r image-requirements.txt
 pip install -r docs/requirements.txt
 pip install --no-build-isolation -e ".[dev]"
 ```

audio-requirements.txt

@@ -0,0 +1 @@
+torchaudio

docs/source/conf.py

@@ -34,7 +34,7 @@
 # -- Project information -----------------------------------------------------
 
 project = "TorchEval"
-copyright = "2023, Meta"
+copyright = "2025, Meta"
 author = "Meta"
 
 # The full version, including alpha/beta/rc tags
@@ -112,7 +112,9 @@ def setup(app):
 
     # In Sphinx 1.8 it was renamed to `add_css_file`, 1.7 and prior it is
     # `add_stylesheet` (deprecated in 1.8).
-    add_css = getattr(app, "add_css_file", getattr(app, "add_stylesheet", None))  # noqa B009
+    add_css = getattr(
+        app, "add_css_file", getattr(app, "add_stylesheet", None)
+    )  # noqa B009
     for css_file in html_css_files:
         add_css(css_file)
     app.set_translator("html", PatchedHTMLTranslator)

docs/source/torcheval.metrics.functional.rst

@@ -25,6 +25,7 @@ Classification Metrics
    binary_accuracy
    binary_auprc
    binary_auroc
+   binary_binned_auprc
    binary_binned_auroc
    binary_binned_precision_recall_curve
    binary_confusion_matrix
@@ -37,6 +38,7 @@ Classification Metrics
    multiclass_accuracy
    multiclass_auprc
    multiclass_auroc
+   multiclass_binned_auprc
    multiclass_binned_auroc
    multiclass_binned_precision_recall_curve
    multiclass_confusion_matrix
@@ -46,6 +48,8 @@ Classification Metrics
    multiclass_recall
    multilabel_accuracy
    multilabel_auprc
+   multilabel_binned_auprc
+   multilabel_binned_precision_recall_curve
    multilabel_precision_recall_curve
    multilabel_recall_at_fixed_precision
    topk_multilabel_accuracy
@@ -72,6 +76,8 @@ Ranking Metrics
    num_collisions
    reciprocal_rank
    weighted_calibration
+   retrieval_precision
+   retrieval_recall
 
 Regression Metrics
 -------------------------------------------------------------------
@@ -83,6 +89,15 @@ Regression Metrics
    mean_squared_error
    r2_score
 
+Statistical Metrics
+-------------------------------------------------------------------
+
+.. autosummary::
+   :toctree: generated
+   :nosignatures:
+
+   wasserstein_1d
+
 Text Metrics
 -------------------------------------------------------------------
 
@@ -95,3 +110,4 @@ Text Metrics
    word_error_rate
    word_information_preserved
    word_information_lost
+

docs/source/torcheval.metrics.rst

@@ -13,6 +13,7 @@ Aggregation Metrics
 
    AUC
    Cat
+   Covariance
    Max
    Mean
    Min
@@ -39,6 +40,7 @@ Classification Metrics
    BinaryAUPRC
    BinaryAUROC
    BinaryBinnedAUROC
+   BinaryBinnedAUPRC
    BinaryBinnedPrecisionRecallCurve
    BinaryConfusionMatrix
    BinaryF1Score
@@ -50,6 +52,7 @@ Classification Metrics
    MulticlassAccuracy
    MulticlassAUPRC
    MulticlassAUROC
+   MulticlassBinnedAUPRC
    MulticlassBinnedAUROC
    MulticlassBinnedPrecisionRecallCurve
    MulticlassConfusionMatrix
@@ -59,6 +62,8 @@ Classification Metrics
    MulticlassRecall
    MultilabelAccuracy
    MultilabelAUPRC
+   MultilabelBinnedAUPRC
+   MultilabelBinnedPrecisionRecallCurve
    MultilabelPrecisionRecallCurve
    MultilabelRecallAtFixedPrecision
    TopKMultilabelAccuracy
@@ -84,6 +89,8 @@ Ranking Metrics
    ClickThroughRate
    HitRate
    ReciprocalRank
+   RetrievalPrecision
+   RetrievalRecall
    WeightedCalibration
 
 Regression Metrics
@@ -96,6 +103,15 @@ Regression Metrics
    MeanSquaredError
    R2Score
 
+Statistical Metrics
+-------------------------------------------------------------------
+
+.. autosummary::
+   :toctree: generated
+   :nosignatures:
+
+   Wasserstein1D
+
 Text Metrics
 -------------------------------------------------------------------
 
@@ -121,3 +137,4 @@ Windowed Metrics
    WindowedClickThroughRate
    WindowedMeanSquaredError
    WindowedWeightedCalibration
+

setup.py

@@ -76,6 +76,7 @@ def parse_args() -> argparse.Namespace:
             "Topic :: Scientific/Engineering :: Artificial Intelligence",
         ],
         extras_require={
+            "audio": read_requirements("audio-requirements.txt"),
             "dev": read_requirements("dev-requirements.txt"),
             "image": read_requirements("image-requirements.txt"),
         },

tests/metrics/text/test_word_error_rate.py

@@ -30,7 +30,7 @@ def test_word_error_rate_with_valid_input(self) -> None:
                     ["hello metaverse", "welcome to meta"],
                 ],
             },
-            compute_result=torch.tensor(0.6),
+            compute_result=torch.tensor(0.6, dtype=torch.float64),
             num_total_updates=4,
         )
 

torcheval/metrics/__init__.py

@@ -69,6 +69,8 @@
 )
 from torcheval.metrics.regression import MeanSquaredError, R2Score
 
+from torcheval.metrics.statistical import Wasserstein1D
+
 from torcheval.metrics.text import (
     BLEUScore,
     Perplexity,
@@ -142,6 +144,7 @@
     "StructuralSimilarity",
     "Sum",
     "Throughput",
+    "Wasserstein1D",
     "WeightedCalibration",
     "WindowedBinaryAUROC",
     "WindowedBinaryNormalizedEntropy",

torcheval/metrics/aggregation/mean.py

@@ -16,6 +16,7 @@
 
 from torcheval.metrics.functional.aggregation.mean import _mean_update
 from torcheval.metrics.metric import Metric
+from torcheval.utils.device import largest_float
 
 TMean = TypeVar("TMean")
 
@@ -24,7 +25,7 @@ class Mean(Metric[torch.Tensor]):
     """
     Calculate the weighted mean value of all elements in all the input tensors.
     When weight is not provided, it calculates the unweighted mean.
-    Its functional version is ``torcheval.functional.mean()``.
+    Its functional version is :func:`torcheval.metrics.functional.mean`.
 
     Examples::
 
@@ -58,14 +59,13 @@ def __init__(
         device: torch.device | None = None,
     ) -> None:
         super().__init__(device=device)
+        dtype = largest_float(device)
         # weighted sum of values over the entire state
         self._add_state(
-            "weighted_sum", torch.tensor(0.0, device=self.device, dtype=torch.float64)
+            "weighted_sum", torch.tensor(0.0, device=self.device, dtype=dtype)
         )
         # sum total of weights over the entire state
-        self._add_state(
-            "weights", torch.tensor(0.0, device=self.device, dtype=torch.float64)
-        )
+        self._add_state("weights", torch.tensor(0.0, device=self.device, dtype=dtype))
 
     @torch.inference_mode()
     # pyre-ignore[14]: inconsistent override on *_:Any, **__:Any
@@ -102,7 +102,7 @@ def compute(self: TMean) -> torch.Tensor:
             logging.warning(
                 "There is no weight for the average, no samples with weight have been added (did you ever run update()?)- returning 0.0"
             )
-            return torch.tensor(0.0, dtype=torch.float64)
+            return torch.tensor(0.0, dtype=largest_float(self.device))
         return self.weighted_sum / self.weights
 
     @torch.inference_mode()

torcheval/metrics/aggregation/sum.py

@@ -15,6 +15,7 @@
 
 from torcheval.metrics.functional.aggregation.sum import _sum_update
 from torcheval.metrics.metric import Metric
+from torcheval.utils.device import largest_float
 
 TSum = TypeVar("TSum")
 
@@ -56,7 +57,8 @@ def __init__(
     ) -> None:
         super().__init__(device=device)
         self._add_state(
-            "weighted_sum", torch.tensor(0.0, device=self.device, dtype=torch.float64)
+            "weighted_sum",
+            torch.tensor(0.0, device=self.device, dtype=largest_float(device)),
         )
 
     @torch.inference_mode()

torcheval/metrics/audio/__init__.py

@@ -10,3 +10,4 @@
 
 
 __all__ = ["FrechetAudioDistance"]
+__doc_name__ = "Audio Metrics"

torcheval/metrics/classification/auroc.py

@@ -21,6 +21,7 @@
     _multiclass_auroc_update_input_check,
 )
 from torcheval.metrics.metric import Metric
+from torcheval.utils.device import largest_float
 
 try:
     import fbgemm_gpu.metrics  # noqa
@@ -113,7 +114,7 @@ def update(
         target = target.to(self.device)
 
         if weight is None:
-            weight = torch.ones_like(input, dtype=torch.double)
+            weight = torch.ones_like(input, dtype=largest_float(self.device))
         _binary_auroc_update_input_check(input, target, self.num_tasks, weight)
         self.inputs.append(input)
         self.targets.append(target)

torcheval/metrics/classification/binary_normalized_entropy.py

@@ -18,6 +18,7 @@
     _binary_normalized_entropy_update,
 )
 from torcheval.metrics.metric import Metric
+from torcheval.utils.device import largest_float
 
 TNormalizedEntropy = TypeVar("TNormalizedEntropy")
 
@@ -76,17 +77,18 @@ def __init__(
                 "`num_tasks` value should be greater than and equal to 1, but received {num_tasks}. "
             )
         self.num_tasks = num_tasks
+        dtype = largest_float(device)
         self._add_state(
             "total_entropy",
-            torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device),
+            torch.zeros(self.num_tasks, dtype=dtype, device=self.device),
         )
         self._add_state(
             "num_examples",
-            torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device),
+            torch.zeros(self.num_tasks, dtype=dtype, device=self.device),
         )
         self._add_state(
             "num_positive",
-            torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device),
+            torch.zeros(self.num_tasks, dtype=dtype, device=self.device),
         )
 
     @torch.inference_mode()

torcheval/metrics/functional/__init__.py

@@ -53,6 +53,7 @@
     weighted_calibration,
 )
 from torcheval.metrics.functional.regression import mean_squared_error, r2_score
+from torcheval.metrics.functional.statistical import wasserstein_1d
 from torcheval.metrics.functional.text import (
     bleu_score,
     perplexity,
@@ -110,6 +111,7 @@
     "sum",
     "throughput",
     "topk_multilabel_accuracy",
+    "wasserstein_1d",
     "weighted_calibration",
     "word_error_rate",
     "word_information_preserved",

torcheval/metrics/functional/aggregation/mean.py

@@ -17,7 +17,7 @@ def mean(
 ) -> torch.Tensor:
     """
     Compute weighted mean. When weight is not provided, it calculates the unweighted mean.
-    Its class version is ``torcheval.metrics.Mean``.
+    Its class version is :obj:`torcheval.metrics.Mean`.
 
     weighted_mean = sum(weight * input) / sum(weight)
 

torcheval/metrics/functional/aggregation/sum.py

@@ -17,7 +17,7 @@ def sum(
 ) -> torch.Tensor:
     """
     Compute weighted sum. When weight is not provided, it calculates the unweighted sum.
-    Its class version is ``torcheval.metrics.Sum``.
+    Its class version is :obj:`torcheval.metrics.Sum`.
 
     Args:
         input (Tensor): Tensor of input values.

torcheval/metrics/functional/aggregation/throughput.py

@@ -17,7 +17,7 @@ def throughput(
 ) -> torch.Tensor:
     """
     Calculate the throughput value which is the number of elements processed per second.
-    Its class version is ``torcheval.metrics.Throughput``.
+    Its class version is :obj:`torcheval.metrics.Throughput`.
 
     Args:
         num_processed (int): Number of items processed.

torcheval/metrics/functional/classification/accuracy.py

@@ -58,7 +58,7 @@ def multiclass_accuracy(
 ) -> torch.Tensor:
     """
     Compute accuracy score, which is the frequency of input matching target.
-    Its class version is ``torcheval.metrics.MultiClassAccuracy``.
+    Its class version is :obj:`torcheval.metrics.MulticlassAccuracy`.
     See also :func:`binary_accuracy <torcheval.metrics.functional.binary_accuracy>`, :func:`multilabel_accuracy <torcheval.metrics.functional.multilabel_accuracy>`, :func:`topk_multilabel_accuracy <torcheval.metrics.functional.topk_multilabel_accuracy>`
 
     Args:
@@ -116,7 +116,7 @@ def multilabel_accuracy(
 ) -> torch.Tensor:
     """
     Compute multilabel accuracy score, which is the frequency of input matching target.
-    Its class version is ``torcheval.metrics.MultilabelAccuracy``.
+    Its class version is :obj:`torcheval.metrics.MultilabelAccuracy`.
     See also :func:`binary_accuracy <torcheval.metrics.functional.binary_accuracy>`, :func:`multiclass_accuracy <torcheval.metrics.functional.multiclass_accuracy>`, :func:`topk_multilabel_accuracy <torcheval.metrics.functional.topk_multilabel_accuracy>`
 
     Args:
@@ -187,7 +187,7 @@ def topk_multilabel_accuracy(
 ) -> torch.Tensor:
     """
     Compute multilabel accuracy score, which is the frequency of the top k label predicted matching target.
-    Its class version is ``torcheval.metrics.TopKMultilabelAccuracy``.
+    Its class version is :obj:`torcheval.metrics.TopKMultilabelAccuracy`.
     See also :func:`binary_accuracy <torcheval.metrics.functional.binary_accuracy>`, :func:`multiclass_accuracy <torcheval.metrics.functional.multiclass_accuracy>`, :func:`multilabel_accuracy <torcheval.metrics.functional.multilabel_accuracy>`
 
     Args:
@@ -272,9 +272,11 @@ def _multiclass_accuracy_update(
         return num_correct, num_total
 
     # pyre-ignore[6]: expect int got Optional[int] for num_classes
-    num_correct = mask.new_zeros(num_classes).scatter_(0, target, mask, reduce="add")
+    num_correct = mask.new_zeros(num_classes).scatter_add_(0, target, mask)
     # pyre-ignore[6]: expect int got Optional[int] for num_classes
-    num_total = target.new_zeros(num_classes).scatter_(0, target, 1, reduce="add")
+    num_total = target.new_zeros(num_classes).scatter_add_(
+        0, target, torch.ones_like(target)
+    )
     return num_correct, num_total