diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1fade8..6e2ffc6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,9 @@ linting, testing, and building the documentation, run the following: git clone https://github.com/pytorch/torcheval cd torcheval pip install -r requirements.txt +pip install -r audio-requirements.txt pip install -r dev-requirements.txt +pip install -r image-requirements.txt pip install -r docs/requirements.txt pip install --no-build-isolation -e ".[dev]" ``` diff --git a/audio-requirements.txt b/audio-requirements.txt new file mode 100644 index 0000000..3daffcd --- /dev/null +++ b/audio-requirements.txt @@ -0,0 +1 @@ +torchaudio diff --git a/docs/source/conf.py b/docs/source/conf.py index b88219d..57da6cd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -34,7 +34,7 @@ # -- Project information ----------------------------------------------------- project = "TorchEval" -copyright = "2023, Meta" +copyright = "2025, Meta" author = "Meta" # The full version, including alpha/beta/rc tags @@ -112,7 +112,9 @@ def setup(app): # In Sphinx 1.8 it was renamed to `add_css_file`, 1.7 and prior it is # `add_stylesheet` (deprecated in 1.8). - add_css = getattr(app, "add_css_file", getattr(app, "add_stylesheet", None)) # noqa B009 + add_css = getattr( + app, "add_css_file", getattr(app, "add_stylesheet", None) + ) # noqa B009 for css_file in html_css_files: add_css(css_file) app.set_translator("html", PatchedHTMLTranslator) diff --git a/docs/source/torcheval.metrics.functional.rst b/docs/source/torcheval.metrics.functional.rst index 417d09f..b942ba8 100644 --- a/docs/source/torcheval.metrics.functional.rst +++ b/docs/source/torcheval.metrics.functional.rst @@ -25,6 +25,7 @@ Classification Metrics binary_accuracy binary_auprc binary_auroc + binary_binned_auprc binary_binned_auroc binary_binned_precision_recall_curve binary_confusion_matrix @@ -37,6 +38,7 @@ Classification Metrics multiclass_accuracy multiclass_auprc multiclass_auroc + multiclass_binned_auprc multiclass_binned_auroc multiclass_binned_precision_recall_curve multiclass_confusion_matrix @@ -46,6 +48,8 @@ Classification Metrics multiclass_recall multilabel_accuracy multilabel_auprc + multilabel_binned_auprc + multilabel_binned_precision_recall_curve multilabel_precision_recall_curve multilabel_recall_at_fixed_precision topk_multilabel_accuracy @@ -72,6 +76,8 @@ Ranking Metrics num_collisions reciprocal_rank weighted_calibration + retrieval_precision + retrieval_recall Regression Metrics ------------------------------------------------------------------- @@ -83,6 +89,15 @@ Regression Metrics mean_squared_error r2_score +Statistical Metrics +------------------------------------------------------------------- + +.. autosummary:: + :toctree: generated + :nosignatures: + + wasserstein_1d + Text Metrics ------------------------------------------------------------------- @@ -95,3 +110,4 @@ Text Metrics word_error_rate word_information_preserved word_information_lost + diff --git a/docs/source/torcheval.metrics.rst b/docs/source/torcheval.metrics.rst index 6a7bc7c..21c6759 100644 --- a/docs/source/torcheval.metrics.rst +++ b/docs/source/torcheval.metrics.rst @@ -13,6 +13,7 @@ Aggregation Metrics AUC Cat + Covariance Max Mean Min @@ -39,6 +40,7 @@ Classification Metrics BinaryAUPRC BinaryAUROC BinaryBinnedAUROC + BinaryBinnedAUPRC BinaryBinnedPrecisionRecallCurve BinaryConfusionMatrix BinaryF1Score @@ -50,6 +52,7 @@ Classification Metrics MulticlassAccuracy MulticlassAUPRC MulticlassAUROC + MulticlassBinnedAUPRC MulticlassBinnedAUROC MulticlassBinnedPrecisionRecallCurve MulticlassConfusionMatrix @@ -59,6 +62,8 @@ Classification Metrics MulticlassRecall MultilabelAccuracy MultilabelAUPRC + MultilabelBinnedAUPRC + MultilabelBinnedPrecisionRecallCurve MultilabelPrecisionRecallCurve MultilabelRecallAtFixedPrecision TopKMultilabelAccuracy @@ -84,6 +89,8 @@ Ranking Metrics ClickThroughRate HitRate ReciprocalRank + RetrievalPrecision + RetrievalRecall WeightedCalibration Regression Metrics @@ -96,6 +103,15 @@ Regression Metrics MeanSquaredError R2Score +Statistical Metrics +------------------------------------------------------------------- + +.. autosummary:: + :toctree: generated + :nosignatures: + + Wasserstein1D + Text Metrics ------------------------------------------------------------------- @@ -121,3 +137,4 @@ Windowed Metrics WindowedClickThroughRate WindowedMeanSquaredError WindowedWeightedCalibration + diff --git a/setup.py b/setup.py index cb85a53..f033634 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,7 @@ def parse_args() -> argparse.Namespace: "Topic :: Scientific/Engineering :: Artificial Intelligence", ], extras_require={ + "audio": read_requirements("audio-requirements.txt"), "dev": read_requirements("dev-requirements.txt"), "image": read_requirements("image-requirements.txt"), }, diff --git a/tests/metrics/text/test_word_error_rate.py b/tests/metrics/text/test_word_error_rate.py index bfbfee6..0f3b2c4 100644 --- a/tests/metrics/text/test_word_error_rate.py +++ b/tests/metrics/text/test_word_error_rate.py @@ -30,7 +30,7 @@ def test_word_error_rate_with_valid_input(self) -> None: ["hello metaverse", "welcome to meta"], ], }, - compute_result=torch.tensor(0.6), + compute_result=torch.tensor(0.6, dtype=torch.float64), num_total_updates=4, ) diff --git a/torcheval/metrics/__init__.py b/torcheval/metrics/__init__.py index 6c06b8a..cfba855 100644 --- a/torcheval/metrics/__init__.py +++ b/torcheval/metrics/__init__.py @@ -69,6 +69,8 @@ ) from torcheval.metrics.regression import MeanSquaredError, R2Score +from torcheval.metrics.statistical import Wasserstein1D + from torcheval.metrics.text import ( BLEUScore, Perplexity, @@ -142,6 +144,7 @@ "StructuralSimilarity", "Sum", "Throughput", + "Wasserstein1D", "WeightedCalibration", "WindowedBinaryAUROC", "WindowedBinaryNormalizedEntropy", diff --git a/torcheval/metrics/aggregation/mean.py b/torcheval/metrics/aggregation/mean.py index 0e5e336..08c0d7d 100644 --- a/torcheval/metrics/aggregation/mean.py +++ b/torcheval/metrics/aggregation/mean.py @@ -16,6 +16,7 @@ from torcheval.metrics.functional.aggregation.mean import _mean_update from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TMean = TypeVar("TMean") @@ -24,7 +25,7 @@ class Mean(Metric[torch.Tensor]): """ Calculate the weighted mean value of all elements in all the input tensors. When weight is not provided, it calculates the unweighted mean. - Its functional version is ``torcheval.functional.mean()``. + Its functional version is :func:`torcheval.metrics.functional.mean`. Examples:: @@ -58,14 +59,13 @@ def __init__( device: torch.device | None = None, ) -> None: super().__init__(device=device) + dtype = largest_float(device) # weighted sum of values over the entire state self._add_state( - "weighted_sum", torch.tensor(0.0, device=self.device, dtype=torch.float64) + "weighted_sum", torch.tensor(0.0, device=self.device, dtype=dtype) ) # sum total of weights over the entire state - self._add_state( - "weights", torch.tensor(0.0, device=self.device, dtype=torch.float64) - ) + self._add_state("weights", torch.tensor(0.0, device=self.device, dtype=dtype)) @torch.inference_mode() # pyre-ignore[14]: inconsistent override on *_:Any, **__:Any @@ -102,7 +102,7 @@ def compute(self: TMean) -> torch.Tensor: logging.warning( "There is no weight for the average, no samples with weight have been added (did you ever run update()?)- returning 0.0" ) - return torch.tensor(0.0, dtype=torch.float64) + return torch.tensor(0.0, dtype=largest_float(self.device)) return self.weighted_sum / self.weights @torch.inference_mode() diff --git a/torcheval/metrics/aggregation/sum.py b/torcheval/metrics/aggregation/sum.py index 982dd12..14940c3 100644 --- a/torcheval/metrics/aggregation/sum.py +++ b/torcheval/metrics/aggregation/sum.py @@ -15,6 +15,7 @@ from torcheval.metrics.functional.aggregation.sum import _sum_update from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TSum = TypeVar("TSum") @@ -56,7 +57,8 @@ def __init__( ) -> None: super().__init__(device=device) self._add_state( - "weighted_sum", torch.tensor(0.0, device=self.device, dtype=torch.float64) + "weighted_sum", + torch.tensor(0.0, device=self.device, dtype=largest_float(device)), ) @torch.inference_mode() diff --git a/torcheval/metrics/audio/__init__.py b/torcheval/metrics/audio/__init__.py index 0763d17..100a6d7 100644 --- a/torcheval/metrics/audio/__init__.py +++ b/torcheval/metrics/audio/__init__.py @@ -10,3 +10,4 @@ __all__ = ["FrechetAudioDistance"] +__doc_name__ = "Audio Metrics" diff --git a/torcheval/metrics/classification/auroc.py b/torcheval/metrics/classification/auroc.py index 5e41a55..bb2cdbc 100644 --- a/torcheval/metrics/classification/auroc.py +++ b/torcheval/metrics/classification/auroc.py @@ -21,6 +21,7 @@ _multiclass_auroc_update_input_check, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float try: import fbgemm_gpu.metrics # noqa @@ -113,7 +114,7 @@ def update( target = target.to(self.device) if weight is None: - weight = torch.ones_like(input, dtype=torch.double) + weight = torch.ones_like(input, dtype=largest_float(self.device)) _binary_auroc_update_input_check(input, target, self.num_tasks, weight) self.inputs.append(input) self.targets.append(target) diff --git a/torcheval/metrics/classification/binary_normalized_entropy.py b/torcheval/metrics/classification/binary_normalized_entropy.py index 7742551..b1e9c35 100644 --- a/torcheval/metrics/classification/binary_normalized_entropy.py +++ b/torcheval/metrics/classification/binary_normalized_entropy.py @@ -18,6 +18,7 @@ _binary_normalized_entropy_update, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TNormalizedEntropy = TypeVar("TNormalizedEntropy") @@ -76,17 +77,18 @@ def __init__( "`num_tasks` value should be greater than and equal to 1, but received {num_tasks}. " ) self.num_tasks = num_tasks + dtype = largest_float(device) self._add_state( "total_entropy", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "num_examples", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "num_positive", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) @torch.inference_mode() diff --git a/torcheval/metrics/functional/__init__.py b/torcheval/metrics/functional/__init__.py index 288416b..4b20c51 100644 --- a/torcheval/metrics/functional/__init__.py +++ b/torcheval/metrics/functional/__init__.py @@ -53,6 +53,7 @@ weighted_calibration, ) from torcheval.metrics.functional.regression import mean_squared_error, r2_score +from torcheval.metrics.functional.statistical import wasserstein_1d from torcheval.metrics.functional.text import ( bleu_score, perplexity, @@ -110,6 +111,7 @@ "sum", "throughput", "topk_multilabel_accuracy", + "wasserstein_1d", "weighted_calibration", "word_error_rate", "word_information_preserved", diff --git a/torcheval/metrics/functional/aggregation/mean.py b/torcheval/metrics/functional/aggregation/mean.py index 0c58682..51c669d 100644 --- a/torcheval/metrics/functional/aggregation/mean.py +++ b/torcheval/metrics/functional/aggregation/mean.py @@ -17,7 +17,7 @@ def mean( ) -> torch.Tensor: """ Compute weighted mean. When weight is not provided, it calculates the unweighted mean. - Its class version is ``torcheval.metrics.Mean``. + Its class version is :obj:`torcheval.metrics.Mean`. weighted_mean = sum(weight * input) / sum(weight) diff --git a/torcheval/metrics/functional/aggregation/sum.py b/torcheval/metrics/functional/aggregation/sum.py index 0328ce0..a77bb2a 100644 --- a/torcheval/metrics/functional/aggregation/sum.py +++ b/torcheval/metrics/functional/aggregation/sum.py @@ -17,7 +17,7 @@ def sum( ) -> torch.Tensor: """ Compute weighted sum. When weight is not provided, it calculates the unweighted sum. - Its class version is ``torcheval.metrics.Sum``. + Its class version is :obj:`torcheval.metrics.Sum`. Args: input (Tensor): Tensor of input values. diff --git a/torcheval/metrics/functional/aggregation/throughput.py b/torcheval/metrics/functional/aggregation/throughput.py index 73543e7..27fb2d7 100644 --- a/torcheval/metrics/functional/aggregation/throughput.py +++ b/torcheval/metrics/functional/aggregation/throughput.py @@ -17,7 +17,7 @@ def throughput( ) -> torch.Tensor: """ Calculate the throughput value which is the number of elements processed per second. - Its class version is ``torcheval.metrics.Throughput``. + Its class version is :obj:`torcheval.metrics.Throughput`. Args: num_processed (int): Number of items processed. diff --git a/torcheval/metrics/functional/classification/accuracy.py b/torcheval/metrics/functional/classification/accuracy.py index 25ec424..d08c65f 100644 --- a/torcheval/metrics/functional/classification/accuracy.py +++ b/torcheval/metrics/functional/classification/accuracy.py @@ -58,7 +58,7 @@ def multiclass_accuracy( ) -> torch.Tensor: """ Compute accuracy score, which is the frequency of input matching target. - Its class version is ``torcheval.metrics.MultiClassAccuracy``. + Its class version is :obj:`torcheval.metrics.MulticlassAccuracy`. See also :func:`binary_accuracy `, :func:`multilabel_accuracy `, :func:`topk_multilabel_accuracy ` Args: @@ -116,7 +116,7 @@ def multilabel_accuracy( ) -> torch.Tensor: """ Compute multilabel accuracy score, which is the frequency of input matching target. - Its class version is ``torcheval.metrics.MultilabelAccuracy``. + Its class version is :obj:`torcheval.metrics.MultilabelAccuracy`. See also :func:`binary_accuracy `, :func:`multiclass_accuracy `, :func:`topk_multilabel_accuracy ` Args: @@ -187,7 +187,7 @@ def topk_multilabel_accuracy( ) -> torch.Tensor: """ Compute multilabel accuracy score, which is the frequency of the top k label predicted matching target. - Its class version is ``torcheval.metrics.TopKMultilabelAccuracy``. + Its class version is :obj:`torcheval.metrics.TopKMultilabelAccuracy`. See also :func:`binary_accuracy `, :func:`multiclass_accuracy `, :func:`multilabel_accuracy ` Args: @@ -272,9 +272,11 @@ def _multiclass_accuracy_update( return num_correct, num_total # pyre-ignore[6]: expect int got Optional[int] for num_classes - num_correct = mask.new_zeros(num_classes).scatter_(0, target, mask, reduce="add") + num_correct = mask.new_zeros(num_classes).scatter_add_(0, target, mask) # pyre-ignore[6]: expect int got Optional[int] for num_classes - num_total = target.new_zeros(num_classes).scatter_(0, target, 1, reduce="add") + num_total = target.new_zeros(num_classes).scatter_add_( + 0, target, torch.ones_like(target) + ) return num_correct, num_total diff --git a/torcheval/metrics/functional/classification/auprc.py b/torcheval/metrics/functional/classification/auprc.py index f980bfb7..e972fe1 100644 --- a/torcheval/metrics/functional/classification/auprc.py +++ b/torcheval/metrics/functional/classification/auprc.py @@ -25,7 +25,7 @@ def binary_auprc( ) -> torch.Tensor: r""" Compute AUPRC, also called Average Precision, which is the area under the Precision-Recall Curve, for binary classification. - Its class version is ``torcheval.metrics.BinaryAUPRC``. + Its class version is :obj:`torcheval.metrics.BinaryAUPRC`. Precision is defined as :math:`\frac{T_p}{T_p+F_p}`; it is the probability that a positive prediction from the model is a true positive. Recall is defined as :math:`\frac{T_p}{T_p+F_n}`; it is the probability that a true positive is predicted to be positive by the model. @@ -79,7 +79,7 @@ def multiclass_auprc( ) -> torch.Tensor: r""" Compute AUPRC, also called Average Precision, which is the area under the Precision-Recall Curve, for multiclass classification. - Its class version is ``torcheval.metrics.MulticlassAUPRC``. + Its class version is :obj:`torcheval.metrics.MulticlassAUPRC`. Precision is defined as :math:`\frac{T_p}{T_p+F_p}`; it is the probability that a positive prediction from the model is a true positive. Recall is defined as :math:`\frac{T_p}{T_p+F_n}`; it is the probability that a true positive is predicted to be positive by the model. @@ -159,7 +159,7 @@ def multilabel_auprc( ) -> torch.Tensor: r""" Compute AUPRC, also called Average Precision, which is the area under the Precision-Recall Curve, for multilabel classification. - Its class version is ``torcheval.metrics.MultilabelAUPRC``. + Its class version is :obj:`torcheval.metrics.MultilabelAUPRC`. Precision is defined as :math:`\frac{T_p}{T_p+F_p}`, it is the probability that a positive prediction from the model is a true positive. Recall is defined as :math:`\frac{T_p}{T_p+F_n}`, it is the probability that a true positive is predicted to be positive by the model. diff --git a/torcheval/metrics/functional/classification/auroc.py b/torcheval/metrics/functional/classification/auroc.py index 547cde8..eecdfba 100644 --- a/torcheval/metrics/functional/classification/auroc.py +++ b/torcheval/metrics/functional/classification/auroc.py @@ -21,6 +21,8 @@ except OSError: pass +from torcheval.utils.device import largest_float + @torch.inference_mode() def binary_auroc( @@ -33,7 +35,7 @@ def binary_auroc( ) -> torch.Tensor: """ Compute AUROC, which is the area under the ROC Curve, for binary classification. - Its class version is ``torcheval.metrics.BinaryAUROC``. + Its class version is :obj:`torcheval.metrics.BinaryAUROC`. See also :func:`multiclass_auroc ` Args: @@ -148,7 +150,7 @@ def _binary_auroc_compute_jit( auroc = torch.where( factor == 0, 0.5, - torch.trapz(cum_tp, cum_fp).double() / factor, + torch.trapz(cum_tp, cum_fp).type(largest_float(target.device)) / factor, ) return auroc @@ -162,7 +164,7 @@ def _binary_auroc_compute( if use_fbgemm: assert input.is_cuda and target.is_cuda, "Tensors have to be on GPU" # auroc does not have weight - weight = torch.ones_like(input, dtype=torch.double) + weight = torch.ones_like(input, dtype=largest_float(input.device)) num_tasks = 1 if len(input.shape) == 1 else input.shape[0] # FBGEMM AUC is an approximation of AUC. It does not mask data in case # that input values are redundant. For the highly redundant input case, diff --git a/torcheval/metrics/functional/classification/binary_normalized_entropy.py b/torcheval/metrics/functional/classification/binary_normalized_entropy.py index 5b3297e..ded406f 100644 --- a/torcheval/metrics/functional/classification/binary_normalized_entropy.py +++ b/torcheval/metrics/functional/classification/binary_normalized_entropy.py @@ -10,6 +10,8 @@ import torch import torch.nn.functional as F +from torcheval.utils.device import largest_float + @torch.inference_mode() def binary_normalized_entropy( @@ -23,7 +25,7 @@ def binary_normalized_entropy( """ Compute the normalized binary cross entropy between predicted input and ground-truth binary target. - Its class version is ``torcheval.metrics.binary_normalized_entropy`` + Its class version is :obj:`torcheval.metrics.BinaryNormalizedEntropy`. Args: input (Tensor): Predicted unnormalized scores (often referred to as logits) or @@ -70,7 +72,7 @@ def binary_normalized_entropy( ) cross_entropy /= num_examples baseline_entropy = _baseline_update(num_positive, num_examples) - return (cross_entropy / baseline_entropy).double() + return (cross_entropy / baseline_entropy).type(largest_float(target.device)) def _binary_normalized_entropy_update( @@ -98,19 +100,23 @@ def _update( cross_entropy = F.binary_cross_entropy( input, target, weight, reduction="none" ).sum(dim=-1) - weight = target.new_ones(target.size()) * 1.0 if weight is None else weight - num_examples = torch.sum(weight, dim=-1).double() - num_positive = torch.sum(weight * target, dim=-1).double() + dtype = largest_float(target.device) + weight = ( + torch.ones_like(target, dtype=dtype) if weight is None else weight.type(dtype) + ) + num_examples = torch.sum(weight, dim=-1) + num_positive = torch.sum(weight * target, dim=-1) return cross_entropy, num_positive, num_examples def _baseline_update( num_positive: torch.Tensor, num_examples: torch.Tensor ) -> torch.Tensor: + dtype = num_positive.dtype base_pos_rate = torch.clamp( (num_positive / num_examples), - min=torch.finfo(torch.float64).eps, - max=1 - torch.finfo(torch.float64).eps, + min=torch.finfo(dtype).eps, + max=1 - torch.finfo(dtype).eps, ) baseline_entropy = -base_pos_rate * torch.log(base_pos_rate) - ( 1 - base_pos_rate diff --git a/torcheval/metrics/functional/classification/binned_auprc.py b/torcheval/metrics/functional/classification/binned_auprc.py index 9d720c8..5419d8c 100644 --- a/torcheval/metrics/functional/classification/binned_auprc.py +++ b/torcheval/metrics/functional/classification/binned_auprc.py @@ -35,7 +35,7 @@ def binary_binned_auprc( ) -> tuple[torch.Tensor, torch.Tensor]: """ Binned Version of AUPRC, which is the area under the AUPRC Curve, for binary classification. - Its class version is ``torcheval.metrics.BinaryBinnedAUPRC``. + Its class version is :obj:`torcheval.metrics.BinaryBinnedAUPRC`. Computation is done by computing the area under the precision/recall curve; precision and recall are computed for the buckets defined by `threshold`. @@ -179,7 +179,7 @@ def multiclass_binned_auprc( ) -> tuple[torch.Tensor, torch.Tensor]: """ Binned Version of AUPRC, which is the area under the AUPRC Curve, for multiclass classification. - Its class version is ``torcheval.metrics.MulticlassBinnedAUPRC``. + Its class version is :obj:`torcheval.metrics.MulticlassBinnedAUPRC`. Computation is done by computing the area under the precision/recall curve; precision and recall are computed for the buckets defined by `threshold`. @@ -326,7 +326,7 @@ def multilabel_binned_auprc( ) -> tuple[torch.Tensor, torch.Tensor]: """ Binned Version of AUPRC, which is the area under the AUPRC Curve, for multilabel classification. - Its class version is ``torcheval.metrics.MultilabelBinnedAUPRC``. + Its class version is :obj:`torcheval.metrics.MultilabelBinnedAUPRC`. Computation is done by computing the area under the precision/recall curve; precision and recall are computed for the buckets defined by `threshold`. diff --git a/torcheval/metrics/functional/classification/binned_auroc.py b/torcheval/metrics/functional/classification/binned_auroc.py index c6dcf75..3ea115d 100644 --- a/torcheval/metrics/functional/classification/binned_auroc.py +++ b/torcheval/metrics/functional/classification/binned_auroc.py @@ -10,6 +10,7 @@ import torch from torch.nn import functional as F from torcheval.metrics.functional.tensor_utils import _create_threshold_tensor +from torcheval.utils.device import largest_float DEFAULT_NUM_THRESHOLD = 200 @@ -24,7 +25,7 @@ def binary_binned_auroc( ) -> tuple[torch.Tensor, torch.Tensor]: """ Compute AUROC, which is the area under the ROC Curve, for binary classification. - Its class version is ``torcheval.metrics.BinaryBinnedAUROC``. + Its class version is :obj:`torcheval.metrics.BinaryBinnedAUROC`. See also :func:`multiclass_binned_auroc ` Args: @@ -133,7 +134,7 @@ def _binary_binned_auroc_compute( auroc = torch.where( factor == 0, 0.5, - torch.trapz(cum_tp, cum_fp).double() / factor, + torch.trapz(cum_tp, cum_fp).type(largest_float(target.device)) / factor, ) return auroc, threshold diff --git a/torcheval/metrics/functional/classification/binned_precision_recall_curve.py b/torcheval/metrics/functional/classification/binned_precision_recall_curve.py index d5023ab..9d89169 100644 --- a/torcheval/metrics/functional/classification/binned_precision_recall_curve.py +++ b/torcheval/metrics/functional/classification/binned_precision_recall_curve.py @@ -15,6 +15,7 @@ _multilabel_precision_recall_curve_update_input_check, ) from torcheval.metrics.functional.tensor_utils import _create_threshold_tensor +from torcheval.utils.device import largest_float @torch.inference_mode() @@ -26,7 +27,7 @@ def binary_binned_precision_recall_curve( ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ Compute precision recall curve with given thresholds. - Its class version is ``torcheval.metrics.BinaryBinnedPrecisionRecallCurve``. + Its class version is :obj:`torcheval.metrics.BinaryBinnedPrecisionRecallCurve`. See also :func:`multiclass_binned_precision_recall_curve ` Args: @@ -93,7 +94,7 @@ def _update( # (index, target) values, stored as 2 * index + target index_target = ( 2 * (torch.searchsorted(threshold, input, right=True) - 1) + target - ).type(torch.float64) + ).type(largest_float(target.device)) hist = torch.histc( index_target, bins=2 * num_thresholds, min=0, max=2 * num_thresholds ) @@ -140,7 +141,7 @@ def multiclass_binned_precision_recall_curve( ) -> tuple[list[torch.Tensor], list[torch.Tensor], torch.Tensor]: """ Compute precision recall curve with given thresholds. - Its class version is ``torcheval.metrics.MulticlassBinnedPrecisionRecallCurve``. + Its class version is :obj:`torcheval.metrics.MulticlassBinnedPrecisionRecallCurve`. See also :func:`binary_binned_precision_recall_curve ` Args: @@ -256,6 +257,7 @@ def _multiclass_binned_precision_recall_curve_update_memory( num_samples, num_classes = tuple(input.shape) num_thresholds = len(threshold) + big_float = largest_float(target.device) # false positives are positives_idx[0], true positives are positives_idx[1] # For each j, k we find largest i such that input[j,k] >= threshold[i]. We also need to store whether target[j] == k. # largest_index: (index, class, target) values, stored as 2 * ((num_classes * index) + class) + target @@ -265,7 +267,7 @@ def _multiclass_binned_precision_recall_curve_update_memory( num_classes * (torch.searchsorted(threshold, input, right=True) - 1) + torch.arange(num_classes, device=input.device) ) - ).type(torch.float64) + ).type(big_float) largest_index[range(num_samples), target] += 1 hist = torch.histc( @@ -281,7 +283,7 @@ def _multiclass_binned_precision_recall_curve_update_memory( ) class_counts = torch.histc( - target.type(torch.float64), bins=num_classes, min=0, max=num_classes + target.type(big_float), bins=num_classes, min=0, max=num_classes ).type(target.dtype) # suffix sum: For each threshold index/("true/false" positives) combination, @@ -344,7 +346,7 @@ def multilabel_binned_precision_recall_curve( ) -> tuple[list[torch.Tensor], list[torch.Tensor], torch.Tensor]: """ Compute precision recall curve with given thresholds. - Its class version is ``torcheval.metrics.MultilabelBinnedPrecisionRecallCurve``. + Its class version is :obj:`torcheval.metrics.MultilabelBinnedPrecisionRecallCurve`. See also :func:`binary_binned_precision_recall_curve `, :func:`multiclass_precision_recall_curve ` @@ -448,7 +450,7 @@ def _multilabel_binned_precision_recall_curve_update_memory( """ _multilabel_precision_recall_curve_update_input_check(input, target, num_labels) - num_samples, num_labels = tuple(input.shape) + _, num_labels = tuple(input.shape) num_thresholds = len(threshold) # For each sample index j and label k, we need: @@ -465,7 +467,7 @@ def _multilabel_binned_precision_recall_curve_update_memory( ) hist = torch.histc( - largest_index.type(torch.float64), + largest_index.type(largest_float(target.device)), bins=2 * num_thresholds * num_labels, min=0, max=2 * num_thresholds * num_labels, diff --git a/torcheval/metrics/functional/classification/f1_score.py b/torcheval/metrics/functional/classification/f1_score.py index 2672743..de9fb5e 100644 --- a/torcheval/metrics/functional/classification/f1_score.py +++ b/torcheval/metrics/functional/classification/f1_score.py @@ -62,7 +62,7 @@ def multiclass_f1_score( Compute f1 score, which is defined as the harmonic mean of precision and recall. We convert NaN to zero when f1 score is NaN. This happens when either precision or recall is NaN or when both precision and recall are zero. - Its class version is ``torcheval.metrics.MultiClassF1Score``. + Its class version is :obj:`torcheval.metrics.MulticlassF1Score`. See also :func:`binary_f1_score ` Args: @@ -181,14 +181,14 @@ def _update( # Add this line to bypass torch jit datatype checking assert isinstance(num_classes, int) - num_label = torch.zeros(num_classes, device=target.device).scatter_( - 0, target, 1, reduce="add" + num_label = target.new_zeros(num_classes).scatter_add_( + 0, target, torch.ones_like(target) ) - num_prediction = torch.zeros(num_classes, device=target.device).scatter_( - 0, input, 1, reduce="add" + num_prediction = target.new_zeros(num_classes).scatter_add_( + 0, input, torch.ones_like(input) ) - num_tp = torch.zeros(num_classes, device=target.device).scatter_( - 0, target[input == target], 1, reduce="add" + num_tp = target.new_zeros(num_classes).scatter_add_( + 0, target[input == target], torch.ones_like(target) ) return num_tp, num_label, num_prediction diff --git a/torcheval/metrics/functional/classification/precision.py b/torcheval/metrics/functional/classification/precision.py index 0452fa3..c8eb543 100644 --- a/torcheval/metrics/functional/classification/precision.py +++ b/torcheval/metrics/functional/classification/precision.py @@ -24,7 +24,7 @@ def binary_precision( """ Compute precision score for binary classification class, which is calculated as the ratio between the number of true positives (TP) and the total number of predicted positives (TP + FP). - Its class version is ``torcheval.metrics.BinaryPrecision``. + Its class version is :obj:`torcheval.metrics.BinaryPrecision`. See also :func:`multiclass_precision ` Args: @@ -64,7 +64,7 @@ def multiclass_precision( """ Compute precision score, which is the ratio of the true positives (TP) and the total number of points classified as positives (TP + FP). - Its class version is ``torcheval.metrics.MultiClassPrecision``. + Its class version is :obj:`torcheval.metrics.MulticlassPrecision`. See also :func:`binary_precision ` Args: @@ -129,12 +129,14 @@ def _precision_update( num_fp = (input != target).sum() return num_tp, num_fp, torch.tensor(0.0) - num_label = target.new_zeros(num_classes).scatter_(0, target, 1, reduce="add") - num_tp = target.new_zeros(num_classes).scatter_( - 0, target[input == target], 1, reduce="add" + num_label = target.new_zeros(num_classes).scatter_add_( + 0, target, torch.ones_like(target) ) - num_fp = target.new_zeros(num_classes).scatter_( - 0, input[input != target], 1, reduce="add" + num_tp = target.new_zeros(num_classes).scatter_add_( + 0, target[input == target], torch.ones_like(target) + ) + num_fp = target.new_zeros(num_classes).scatter_add_( + 0, input[input != target], torch.ones_like(target) ) return num_tp, num_fp, num_label diff --git a/torcheval/metrics/functional/classification/precision_recall_curve.py b/torcheval/metrics/functional/classification/precision_recall_curve.py index 68f88e0..4b21aa4 100644 --- a/torcheval/metrics/functional/classification/precision_recall_curve.py +++ b/torcheval/metrics/functional/classification/precision_recall_curve.py @@ -25,7 +25,7 @@ def binary_precision_recall_curve( binary classification tasks. If a class is missing from the target tensor, its recall values are set to 1.0. - Its class version is ``torcheval.metrics.BinaryPrecisionRecallCurve``. + Its class version is :obj:`torcheval.metrics.BinaryPrecisionRecallCurve`. See also :func:`multiclass_precision_recall_curve `, :func:`multilabel_precision_recall_curve ` Args: @@ -103,7 +103,7 @@ def multiclass_precision_recall_curve( multi-class classification tasks. If a class is missing from the target tensor, its recall values are set to 1.0. - Its class version is ``torcheval.metrics.MulticlassPrecisionRecallCurve``. + Its class version is :obj:`torcheval.metrics.MulticlassPrecisionRecallCurve`. See also :func:`binary_precision_recall_curve `, :func:`multilabel_precision_recall_curve ` Args: @@ -243,7 +243,7 @@ def multilabel_precision_recall_curve( multi-label classification tasks. If there are no samples for a label in the target tensor, its recall values are set to 1.0. - Its class version is ``torcheval.metrics.MultilabelPrecisionRecallCurve``. + Its class version is :obj:`torcheval.metrics.MultilabelPrecisionRecallCurve`. See also :func:`binary_precision_recall_curve `, :func:`multiclass_precision_recall_curve ` Args: diff --git a/torcheval/metrics/functional/classification/recall.py b/torcheval/metrics/functional/classification/recall.py index 4aabe88..2ba5392 100644 --- a/torcheval/metrics/functional/classification/recall.py +++ b/torcheval/metrics/functional/classification/recall.py @@ -21,7 +21,7 @@ def binary_recall( """ Compute recall score for binary classification class, which is calculated as the ratio between the number of true positives (TP) and the total number of actual positives (TP + FN). - Its class version is ``torcheval.metrics.BinaryRecall``. + Its class version is :obj:`torcheval.metrics.BinaryRecall`. See also :func:`multiclass_recall ` Args: @@ -104,7 +104,7 @@ def multiclass_recall( """ Compute recall score, which is calculated as the ratio between the number of true positives (TP) and the total number of actual positives (TP + FN). - Its class version is ``torcheval.metrics.MultiClassRecall``. + Its class version is :obj:`torcheval.metrics.MulticlassRecall`. See also :func:`binary_recall ` Args: @@ -173,10 +173,14 @@ def _recall_update( assert isinstance( num_classes, int ), f"`num_classes` must be an integer, but received {num_classes}." - num_labels = target.new_zeros(num_classes).scatter_(0, target, 1, reduce="add") - num_predictions = target.new_zeros(num_classes).scatter_(0, input, 1, reduce="add") - num_tp = target.new_zeros(num_classes).scatter_( - 0, target[input == target], 1, reduce="add" + num_labels = target.new_zeros(num_classes).scatter_add_( + 0, target, torch.ones_like(target) + ) + num_predictions = target.new_zeros(num_classes).scatter_add_( + 0, input, torch.ones_like(input) + ) + num_tp = target.new_zeros(num_classes).scatter_add_( + 0, target[input == target], torch.ones_like(target) ) return num_tp, num_labels, num_predictions diff --git a/torcheval/metrics/functional/classification/recall_at_fixed_precision.py b/torcheval/metrics/functional/classification/recall_at_fixed_precision.py index 77572f0..ee1b6a9 100644 --- a/torcheval/metrics/functional/classification/recall_at_fixed_precision.py +++ b/torcheval/metrics/functional/classification/recall_at_fixed_precision.py @@ -28,7 +28,7 @@ def binary_recall_at_fixed_precision( Returns the highest possible recall value given the minimum precision for binary classification tasks. - Its class version is ``torcheval.metrics.BinaryRecallAtFixedPrecision``. + Its class version is :obj:`torcheval.metrics.BinaryRecallAtFixedPrecision`. See also :func:`multilabel_recall_at_fixed_precision ` Args: @@ -83,7 +83,7 @@ def multilabel_recall_at_fixed_precision( classification tasks. The maximum recall computation for each label is equivalent to _binary_recall_at_fixed_precision_compute in binary_recall_at_fixed_precision. - Its class version is ``torcheval.metrics.MultilabelRecallAtFixedPrecision``. + Its class version is :obj:`torcheval.metrics.MultilabelRecallAtFixedPrecision`. See also :func:`binary_recall_at_fixed_precision ` Args: diff --git a/torcheval/metrics/functional/image/psnr.py b/torcheval/metrics/functional/image/psnr.py index 71117da..8ee81aa 100644 --- a/torcheval/metrics/functional/image/psnr.py +++ b/torcheval/metrics/functional/image/psnr.py @@ -18,7 +18,7 @@ def peak_signal_noise_ratio( ) -> torch.Tensor: """ Compute the peak signal-to-noise ratio between two images. - It's class version is `torcheval.metrics.PeakSignalNoiseRatio` + It's class version is :obj:`torcheval.metrics.PeakSignalNoiseRatio`. Args: input (Tensor): Input image ``(N, C, H, W)``. diff --git a/torcheval/metrics/functional/statistical/wasserstein.py b/torcheval/metrics/functional/statistical/wasserstein.py index 182aed5..e528e9d 100644 --- a/torcheval/metrics/functional/statistical/wasserstein.py +++ b/torcheval/metrics/functional/statistical/wasserstein.py @@ -21,7 +21,7 @@ def wasserstein_1d( The Wasserstein distance between two distributions is intuitively the minimum weight of soil (times distance moved) that would need to be moved - if the two distributions were represented by two piles of soil. + if the two distributions were represented by two piles of soil. [1]_ Args ---------- @@ -70,7 +70,7 @@ def wasserstein_1d( ---------- .. [1] "Wasserstein metric", https://en.wikipedia.org/wiki/Wasserstein_metric .. [2] Ramdas, Garcia, Cuturi "On Wasserstein Two Sample Testing and Related - Families of Nonparametric Tests" (2015). :arXiv:`1509.02237`. + Families of Nonparametric Tests" (2015), https://arxiv.org/abs/1509.02237 Examples -------- diff --git a/torcheval/metrics/functional/text/bleu.py b/torcheval/metrics/functional/text/bleu.py index a019c21..2f70e1e 100644 --- a/torcheval/metrics/functional/text/bleu.py +++ b/torcheval/metrics/functional/text/bleu.py @@ -21,7 +21,7 @@ def bleu_score( ) -> torch.Tensor: """ Compute BLEU score given translations and references for each translation. - Its class version is ``torcheval.metrics.texBLEUScore``. + Its class version is :obj:`torcheval.metrics.BLEUScore`. Args: input: Translations to score. @@ -32,7 +32,7 @@ def bleu_score( Examples: >>> import torch - >>> from torcheval.metrics.functional.text import bleu + >>> from torcheval.metrics.functional import bleu_score >>> candidates = ["the squirrel is eating the nut"] >>> references = [["a squirrel is eating a nut", "the squirrel is eating a tasty nut"]] >>> bleu_score(candidates, references, n_gram=4) diff --git a/torcheval/metrics/functional/text/helper.py b/torcheval/metrics/functional/text/helper.py index 493ee5e..f287fae 100644 --- a/torcheval/metrics/functional/text/helper.py +++ b/torcheval/metrics/functional/text/helper.py @@ -7,12 +7,16 @@ # pyre-strict +from typing import List, Tuple, Union + import torch +from torcheval.utils.device import largest_float + def _edit_distance( - prediction_tokens: list[str], - reference_tokens: list[str], + prediction_tokens: List[str], + reference_tokens: List[str], ) -> int: """ Dynamic programming algorithm to compute the edit distance between two word sequences. @@ -36,24 +40,27 @@ def _edit_distance( def _get_errors_and_totals( - input: str | list[str], - target: str | list[str], -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + input: Union[str, List[str]], + target: Union[str, List[str]], + device: torch.device, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """ Calculate the edit distance, max length and lengths of predicted and reference word sequences. Args: input (str, List[str]): Predicted word sequence(s) to score as a string or list of strings. target (str, List[str]): Reference word sequence(s) as a string or list of strings. + device: The device to allocate tensors on """ if isinstance(input, str): input = [input] if isinstance(target, str): target = [target] - max_total = torch.tensor(0.0, dtype=torch.float64) - errors = torch.tensor(0.0, dtype=torch.float64) - target_total = torch.tensor(0.0, dtype=torch.float64) - input_total = torch.tensor(0.0, dtype=torch.float64) + dtype = largest_float(device) + max_total = torch.tensor(0.0, dtype=dtype, device=device) + errors = torch.tensor(0.0, dtype=dtype, device=device) + target_total = torch.tensor(0.0, dtype=dtype, device=device) + input_total = torch.tensor(0.0, dtype=dtype, device=device) for ipt, tgt in zip(input, target): input_tokens = ipt.split() target_tokens = tgt.split() diff --git a/torcheval/metrics/functional/text/perplexity.py b/torcheval/metrics/functional/text/perplexity.py index a60802a..bc3c243 100644 --- a/torcheval/metrics/functional/text/perplexity.py +++ b/torcheval/metrics/functional/text/perplexity.py @@ -11,6 +11,8 @@ import torch import torch.nn.functional as F +from torcheval.utils.device import largest_float + @torch.inference_mode() def perplexity( @@ -90,7 +92,8 @@ def _perplexity_update( _perplexity_input_check(input, target, ignore_index) - probs = input.reshape(-1, input.shape[-1]) + dtype = largest_float(target.device) + probs = input.reshape(-1, input.shape[-1]).type(dtype) probs = F.softmax(probs, dim=1) target = target.reshape(-1) @@ -112,7 +115,7 @@ def _perplexity_compute( sum_log_probs: torch.Tensor, num_total: torch.Tensor, ) -> torch.Tensor: - return torch.exp(sum_log_probs / num_total).double() + return torch.exp(sum_log_probs / num_total) def _perplexity_input_check( diff --git a/torcheval/metrics/functional/text/word_error_rate.py b/torcheval/metrics/functional/text/word_error_rate.py index 8d65392..eca03ba 100644 --- a/torcheval/metrics/functional/text/word_error_rate.py +++ b/torcheval/metrics/functional/text/word_error_rate.py @@ -6,22 +6,28 @@ # pyre-strict +from typing import List, Optional, Tuple, Union import torch +from torcheval.metrics.functional.text.helper import _edit_distance +from torcheval.utils.device import largest_float + @torch.inference_mode() def word_error_rate( - input: str | list[str], - target: str | list[str], + input: Union[str, List[str]], + target: Union[str, List[str]], + device: Optional[torch.device] = None, ) -> torch.Tensor: """ Compute the word error rate of the predicted word sequence(s) with the reference word sequence(s). - Its class version is ``torcheval.metrics.WordErrorRate``. + Its class version is :obj:`torcheval.metrics.text.WordErrorRate`. Args: input (str, List[str]): Predicted word sequence(s) to score as a string or list of strings. target (str, List[str]): Reference word sequence(s) as a string or list of strings. + device: The device to allocate tensors on Examples: @@ -36,28 +42,31 @@ def word_error_rate( >>> word_error_rate(input, target) tensor(0.5) """ - errors, total = _word_error_rate_update(input, target) + errors, total = _word_error_rate_update(input, target, device) return _word_error_rate_compute(errors, total) def _word_error_rate_update( - input: str | list[str], - target: str | list[str], -) -> tuple[torch.Tensor, torch.Tensor]: + input: Union[str, List[str]], + target: Union[str, List[str]], + device: torch.device, +) -> Tuple[torch.Tensor, torch.Tensor]: """ Update the metric state with edit distance and the length of the reference sequence. Args: input (str, List[str]): Predicted word sequence(s) to score as a string or list of strings. target (str, List[str]): Reference word sequence(s) as a string or list of strings. + device: The device to allocate Tensors on """ _word_error_rate_input_check(input, target) if isinstance(input, str): input = [input] if isinstance(target, str): target = [target] - errors = torch.tensor(0, dtype=torch.float64) - total = torch.tensor(0, dtype=torch.float64) + dtype = largest_float(device) + errors = torch.tensor(0, dtype=dtype, device=device) + total = torch.tensor(0, dtype=dtype, device=device) for ipt, tgt in zip(input, target): ipt_tokens = ipt.split() tgt_tokens = tgt.split() @@ -80,34 +89,9 @@ def _word_error_rate_compute( return errors / total -def _edit_distance( - prediction_tokens: list[str], - reference_tokens: list[str], -) -> int: - """ - Dynamic programming algorithm to compute the edit distance between two word sequences. - - Args: - prediction_tokens (List[str]): A tokenized predicted sentence - reference_tokens (List[str]): A tokenized reference sentence - """ - dp = [[0] * (len(reference_tokens) + 1) for _ in range(len(prediction_tokens) + 1)] - for i in range(len(prediction_tokens) + 1): - dp[i][0] = i - for j in range(len(reference_tokens) + 1): - dp[0][j] = j - for i in range(1, len(prediction_tokens) + 1): - for j in range(1, len(reference_tokens) + 1): - if prediction_tokens[i - 1] == reference_tokens[j - 1]: - dp[i][j] = dp[i - 1][j - 1] - else: - dp[i][j] = min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]) + 1 - return dp[-1][-1] - - def _word_error_rate_input_check( - input: str | list[str], - target: str | list[str], + input: Union[str, List[str]], + target: Union[str, List[str]], ) -> None: if type(input) != type(target): raise ValueError( diff --git a/torcheval/metrics/functional/text/word_information_lost.py b/torcheval/metrics/functional/text/word_information_lost.py index 0d43069..5c89fad 100644 --- a/torcheval/metrics/functional/text/word_information_lost.py +++ b/torcheval/metrics/functional/text/word_information_lost.py @@ -6,6 +6,7 @@ # pyre-strict +from typing import List, Optional, Tuple, Union import torch @@ -13,13 +14,15 @@ def _wil_update( - input: str | list[str], - target: str | list[str], -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + input: Union[str, List[str]], + target: Union[str, List[str]], + device: torch.device, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Update the wil score with the current set of references and predictions. Args: input: Transcription(s) to score as a string or list of strings target: Reference(s) for each speech input as a string or list of strings + device: The device to allocate tensors on Returns: Number of correct words Number of words overall references @@ -29,10 +32,12 @@ def _wil_update( input = [input] if isinstance(target, str): target = [target] - assert ( - len(input) == len(target) + assert len(input) == len( + target ), f"Arguments must contain the same number of strings, but got len(input)={len(input)} and len(target)={len(target)}" - errors, max_total, target_total, input_total = _get_errors_and_totals(input, target) + errors, max_total, target_total, input_total = _get_errors_and_totals( + input, target, device + ) return errors - max_total, target_total, input_total @@ -52,18 +57,20 @@ def _wil_compute( @torch.inference_mode() def word_information_lost( - input: str | list[str], - target: str | list[str], + input: Union[str, List[str]], + target: Union[str, List[str]], + device: Optional[torch.device] = None, ) -> torch.Tensor: """Word Information Lost rate is a metric of the performance of an automatic speech recognition system. This value indicates the percentage of characters that were incorrectly predicted. The lower the value, the better the performance of the ASR system with a Word Information Lost rate of 0 being a perfect score. - Its class version is ``torcheval.metrics.WordInformationLost``. + Its class version is :obj:`torcheval.metrics.text.WordInformationLost`. Args: input: Transcription(s) to score as a string or list of strings target: Reference(s) for each speech input as a string or list of strings + device: The device to allocate Tensors on Returns: Word Information Lost rate Examples: @@ -73,5 +80,5 @@ def word_information_lost( >>> word_information_lost(input, target) tensor(0.6528) """ - correct_total, target_total, preds_total = _wil_update(input, target) + correct_total, target_total, preds_total = _wil_update(input, target, device) return _wil_compute(correct_total, target_total, preds_total) diff --git a/torcheval/metrics/functional/text/word_information_preserved.py b/torcheval/metrics/functional/text/word_information_preserved.py index 8914260..2fd36cb 100644 --- a/torcheval/metrics/functional/text/word_information_preserved.py +++ b/torcheval/metrics/functional/text/word_information_preserved.py @@ -6,6 +6,7 @@ # pyre-strict +from typing import List, Optional, Tuple, Union import torch @@ -14,16 +15,18 @@ @torch.inference_mode() def word_information_preserved( - input: str | list[str], - target: str | list[str], + input: Union[str, List[str]], + target: Union[str, List[str]], + device: Optional[torch.device] = None, ) -> torch.Tensor: """ Compute the word information preserved score of the predicted word sequence(s) against the reference word sequence(s). - Its class version is ``torcheval.metrics.WordInformationPreserved``. + Its class version is :obj:`torcheval.metrics.text.WordInformationPreserved`. Args: input (str, List[str]): Predicted word sequence(s) to score as a string or list of strings. target (str, List[str]): Reference word sequence(s) as a string or list of strings. + device: The device to allocate Tensors on Examples: @@ -39,24 +42,28 @@ def word_information_preserved( tensor(0.3472) """ correct_total, target_total, input_total = _word_information_preserved_update( - input, target + input, target, device ) return _word_information_preserved_compute(correct_total, target_total, input_total) def _word_information_preserved_update( - input: str | list[str], - target: str | list[str], -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + input: Union[str, List[str]], + target: Union[str, List[str]], + device: torch.device, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ Update the word information preserved score with current set of predictions and references. Args: input (str, List[str]): Predicted word sequence(s) to score as a string or list of strings. target (str, List[str]): Reference word sequence(s) as a string or list of strings. + device: The device to allocate tensors on """ _word_information_preserved_input_check(input, target) - errors, max_total, target_total, input_total = _get_errors_and_totals(input, target) + errors, max_total, target_total, input_total = _get_errors_and_totals( + input, target, device + ) return max_total - errors, target_total, input_total @@ -76,8 +83,8 @@ def _word_information_preserved_compute( def _word_information_preserved_input_check( - input: str | list[str], - target: str | list[str], + input: Union[str, List[str]], + target: Union[str, List[str]], ) -> None: if type(input) != type(target): raise ValueError( diff --git a/torcheval/metrics/image/ssim.py b/torcheval/metrics/image/ssim.py index bb12ebf..5166edb 100644 --- a/torcheval/metrics/image/ssim.py +++ b/torcheval/metrics/image/ssim.py @@ -41,14 +41,16 @@ class StructuralSimilarity(Metric[torch.Tensor]): Compute the structural similarity index (SSIM) between two sets of images. Args: - device (torch.device): The device where the computations will be performed. - If None, the default device will be used. + device (torch.device): The device where the computations will be performed. + If None, the default device will be used. """ def __init__( self: TStructuralSimilarity, device: torch.device | None = None, ) -> None: + _validate_torchvision_available() + super().__init__(device=device) self._add_state("mssim_sum", torch.tensor(0, device=device, dtype=torch.float)) diff --git a/torcheval/metrics/ranking/click_through_rate.py b/torcheval/metrics/ranking/click_through_rate.py index 194ee2c..ab97593 100644 --- a/torcheval/metrics/ranking/click_through_rate.py +++ b/torcheval/metrics/ranking/click_through_rate.py @@ -18,6 +18,7 @@ _click_through_rate_update, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TClickThroughRate = TypeVar("TClickThroughRate") @@ -26,7 +27,7 @@ class ClickThroughRate(Metric[torch.Tensor]): """ Compute the click through rate given click events. - Its functional version is ``torcheval.metrics.functional.click_through_rate``. + Its functional version is :func:`torcheval.metrics.functional.click_through_rate`. Args: num_tasks (int): Number of tasks that need weighted_calibration calculation. Default value @@ -67,13 +68,14 @@ def __init__( "`num_tasks` value should be greater than and equal to 1, but received {num_tasks}. " ) self.num_tasks = num_tasks + dtype = largest_float(device) self._add_state( "click_total", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "weight_total", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) @torch.inference_mode() diff --git a/torcheval/metrics/ranking/weighted_calibration.py b/torcheval/metrics/ranking/weighted_calibration.py index b22f413..4150263 100644 --- a/torcheval/metrics/ranking/weighted_calibration.py +++ b/torcheval/metrics/ranking/weighted_calibration.py @@ -17,6 +17,7 @@ _weighted_calibration_update, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TWeightedCalibration = TypeVar("TWeightedCalibration") @@ -68,13 +69,14 @@ def __init__( "`num_tasks` value should be greater than and equal to 1, but received {num_tasks}. " ) self.num_tasks = num_tasks + dtype = largest_float(device) self._add_state( "weighted_input_sum", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "weighted_target_sum", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) @torch.inference_mode() @@ -112,7 +114,7 @@ def compute(self: TWeightedCalibration) -> torch.Tensor: Tensor: The return value of weighted calibration for each task (num_tasks,). """ if torch.any(self.weighted_target_sum == 0.0): - return torch.empty(0) + return torch.empty(0, device=self.device) weighted_calibration = self.weighted_input_sum / self.weighted_target_sum return weighted_calibration diff --git a/torcheval/metrics/text/bleu.py b/torcheval/metrics/text/bleu.py index 6eee054..3e3a3a5 100644 --- a/torcheval/metrics/text/bleu.py +++ b/torcheval/metrics/text/bleu.py @@ -18,6 +18,7 @@ ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TBLEUScore = TypeVar("TBLEUScore") @@ -25,7 +26,7 @@ class BLEUScore(Metric[torch.Tensor]): """ Compute BLEU score (https://en.wikipedia.org/wiki/BLEU) given translations and references. - Its functional version is ``torcheval.metrics.functional.text.bleu``. + Its functional version is :func:`torcheval.metrics.functional.bleu_score`. Args: n_gram: Maximum n-gram to use when computing BLEU score. Can be 1, 2, 3, or 4. @@ -65,19 +66,16 @@ def __init__( self.weights = weights self.n_gram = n_gram - self._add_state( - "input_len", torch.tensor(0.0, dtype=torch.float64, device=device) - ) - self._add_state( - "target_len", torch.tensor(0.0, dtype=torch.float64, device=device) - ) + dtype = largest_float(device) + self._add_state("input_len", torch.tensor(0.0, dtype=dtype, device=device)) + self._add_state("target_len", torch.tensor(0.0, dtype=dtype, device=device)) self._add_state( "matches_by_order", - torch.zeros(n_gram, dtype=torch.float64, device=device), + torch.zeros(n_gram, dtype=dtype, device=device), ) self._add_state( "possible_matches_by_order", - torch.zeros(n_gram, dtype=torch.float64, device=device), + torch.zeros(n_gram, dtype=dtype, device=device), ) @torch.inference_mode() @@ -113,7 +111,9 @@ def compute(self: TBLEUScore) -> torch.Tensor: ``compute()`` is called, return tensor(0.0). """ if torch.sum(self.matches_by_order) == 0: - return torch.tensor(0.0, dtype=torch.float64, device=self.device) + return torch.tensor( + 0.0, dtype=largest_float(self.device), device=self.device + ) return _bleu_score_compute( self.input_len, self.target_len, diff --git a/torcheval/metrics/text/perplexity.py b/torcheval/metrics/text/perplexity.py index 5f63e65..1cf1282 100644 --- a/torcheval/metrics/text/perplexity.py +++ b/torcheval/metrics/text/perplexity.py @@ -18,6 +18,7 @@ _perplexity_update, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TPerplexity = TypeVar("TPerplexity") @@ -28,7 +29,7 @@ class Perplexity(Metric[torch.Tensor]): ppl = exp (sum of negative log likelihood / number of tokens) - Its functional version is ``torcheval.metrics.functional.text.perplexity``. + Its functional version is :func:`torcheval.metrics.functional.perplexity`. Args: ignore_index (Tensor): @@ -80,12 +81,11 @@ def __init__( super().__init__(device=device) self.ignore_index = ignore_index + dtype = largest_float(device) self._add_state( - "sum_log_probs", torch.tensor(0.0, dtype=torch.float64, device=self.device) - ) - self._add_state( - "num_total", torch.tensor(0.0, dtype=torch.float64, device=self.device) + "sum_log_probs", torch.tensor(0.0, dtype=dtype, device=self.device) ) + self._add_state("num_total", torch.tensor(0.0, dtype=dtype, device=self.device)) @torch.inference_mode() # pyre-ignore[14]: `update` overrides method defined in `Metric` inconsistently. @@ -118,7 +118,7 @@ def compute(self: TPerplexity) -> torch.Tensor: If no `update()` calls are made before `compute()` is called, return an empty tensor. """ if self.num_total == 0.0: - return torch.empty(0) + return torch.empty(0, device=self.device) return _perplexity_compute(self.sum_log_probs, self.num_total) @torch.inference_mode() diff --git a/torcheval/metrics/text/word_error_rate.py b/torcheval/metrics/text/word_error_rate.py index 5224bc4..f214d30 100644 --- a/torcheval/metrics/text/word_error_rate.py +++ b/torcheval/metrics/text/word_error_rate.py @@ -18,6 +18,7 @@ _word_error_rate_update, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TWordErrorRate = TypeVar("TWordErrorRate") @@ -53,10 +54,9 @@ def __init__( device: torch.device | None = None, ) -> None: super().__init__(device=device) - self._add_state( - "errors", torch.tensor(0, dtype=torch.float, device=self.device) - ) - self._add_state("total", torch.tensor(0, dtype=torch.float, device=self.device)) + dtype = largest_float(device) + self._add_state("errors", torch.tensor(0, dtype=dtype, device=self.device)) + self._add_state("total", torch.tensor(0, dtype=dtype, device=self.device)) @torch.inference_mode() # pyre-ignore[14]: `update` overrides method defined in `Metric` inconsistently. @@ -72,7 +72,7 @@ def update( input (str, List[str]): Predicted word sequence(s) to score as a string or list of strings. target (str, List[str]): Reference word sequence(s) as a string or list of strings. """ - errors, total = _word_error_rate_update(input, target) + errors, total = _word_error_rate_update(input, target, self.device) self.errors += errors self.total += total return self diff --git a/torcheval/metrics/text/word_information_lost.py b/torcheval/metrics/text/word_information_lost.py index a3b83f5..adcd07a 100644 --- a/torcheval/metrics/text/word_information_lost.py +++ b/torcheval/metrics/text/word_information_lost.py @@ -19,6 +19,7 @@ ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TWordInformationLost = TypeVar("TWordInformationLost") @@ -39,12 +40,13 @@ class WordInformationLost(Metric[torch.Tensor]): Its functional version is :func:`torcheval.metrics.functional.word_information_lost`. Examples: - >>> from torcheval.metrics.text import WordInformationLost + >>> from torcheval.metrics import WordInformationLost >>> preds = ["this is the prediction", "there is an other sample"] >>> target = ["this is the reference", "there is another one"] >>> metric = WordInformationLost() - >>> metric(preds, target) - tensor(0.6528) + >>> metric.update(preds, target) + >>> metric.compute() + tensor(0.6528, dtype=torch.float64) """ def __init__( @@ -53,13 +55,16 @@ def __init__( ) -> None: super().__init__(device=device) self._add_state( - "correct_total", torch.tensor(0.0, dtype=torch.float64, device=self.device) + "correct_total", + torch.tensor(0.0, dtype=largest_float(device), device=self.device), ) self._add_state( - "target_total", torch.tensor(0.0, dtype=torch.float64, device=self.device) + "target_total", + torch.tensor(0.0, dtype=largest_float(device), device=self.device), ) self._add_state( - "preds_total", torch.tensor(0.0, dtype=torch.float64, device=self.device) + "preds_total", + torch.tensor(0.0, dtype=largest_float(device), device=self.device), ) @torch.inference_mode() @@ -74,10 +79,12 @@ def update( input: Transcription(s) to score as a string or list of strings target: Reference(s) for each speech input as a string or list of strings """ - correct_total, target_total, preds_total = _wil_update(input, target) - self.correct_total += correct_total.to(self.device) - self.target_total += target_total.to(self.device) - self.preds_total += preds_total.to(self.device) + correct_total, target_total, preds_total = _wil_update( + input, target, self.device + ) + self.correct_total += correct_total + self.target_total += target_total + self.preds_total += preds_total return self @torch.inference_mode() diff --git a/torcheval/metrics/text/word_information_preserved.py b/torcheval/metrics/text/word_information_preserved.py index 6d514ef..7a97be9 100644 --- a/torcheval/metrics/text/word_information_preserved.py +++ b/torcheval/metrics/text/word_information_preserved.py @@ -18,6 +18,7 @@ _word_information_preserved_update, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TWordInformationPreserved = TypeVar("TWordInformationPreserved") @@ -52,13 +53,16 @@ def __init__( ) -> None: super().__init__(device=device) self._add_state( - "correct_total", torch.tensor(0, dtype=torch.float64, device=self.device) + "correct_total", + torch.tensor(0, dtype=largest_float(device), device=self.device), ) self._add_state( - "input_total", torch.tensor(0, dtype=torch.float64, device=self.device) + "input_total", + torch.tensor(0, dtype=largest_float(device), device=self.device), ) self._add_state( - "target_total", torch.tensor(0, dtype=torch.float64, device=self.device) + "target_total", + torch.tensor(0, dtype=largest_float(device), device=self.device), ) @torch.inference_mode() @@ -76,11 +80,11 @@ def update( target (str, List[str]): Reference word sequence(s) as a string or list of strings. """ correct_total, target_total, input_total = _word_information_preserved_update( - input, target + input, target, self.device ) - self.correct_total += correct_total.to(self.device) - self.target_total += target_total.to(self.device) - self.input_total += input_total.to(self.device) + self.correct_total += correct_total + self.target_total += target_total + self.input_total += input_total return self @torch.inference_mode() diff --git a/torcheval/metrics/window/auroc.py b/torcheval/metrics/window/auroc.py index 0ed98d1..a5f3007 100644 --- a/torcheval/metrics/window/auroc.py +++ b/torcheval/metrics/window/auroc.py @@ -18,6 +18,7 @@ _binary_auroc_update_input_check, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TAUROC = TypeVar("TAUROC") @@ -50,7 +51,10 @@ class WindowedBinaryAUROC(Metric[torch.Tensor]): >>> metric.update(torch.tensor([[0.5, 0.1], [0.3, 0.9]]), torch.tensor([[0.0, 1.0], [0.0, 0.0]])) >>> metric.inputs tensor([[0.1000, 0.3000, 0.8000, 0.3000, 0.5000], - [0.9000, 0.1000, 0.6000, 0.1000, 0.3000]]) + [0.9000, 0.1000, 0.6000, 0.1000, 0.3000]]) + >>> metric.targets + tensor([[1., 0., 1., 1., 0.], + [0., 1., 1., 0., 0.]]) >>> metric.compute() tensor([0.4167, 0.5000]) @@ -108,7 +112,7 @@ def update( or (num_tasks, num_samples). """ if weight is None: - weight = torch.ones_like(input, dtype=torch.double) + weight = torch.ones_like(input, dtype=largest_float(input.device)) _binary_auroc_update_input_check(input, target, self.num_tasks, weight) if input.ndim == 1: input = input.reshape(1, -1) diff --git a/torcheval/metrics/window/click_through_rate.py b/torcheval/metrics/window/click_through_rate.py index f2d53df..da2f7b5 100644 --- a/torcheval/metrics/window/click_through_rate.py +++ b/torcheval/metrics/window/click_through_rate.py @@ -18,6 +18,7 @@ _click_through_rate_update, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TWindowedClickThroughRate = TypeVar("TWindowedClickThroughRate") @@ -32,7 +33,7 @@ class WindowedClickThroughRate( Lifetime value is calculated from all past input and target of `update()` calls. Compute the click through rate given click events. - Its functional version is ``torcheval.metrics.functional.click_through_rate``. + Its functional version is :func:`torcheval.metrics.functional.click_through_rate`. Args: num_tasks (int): Number of tasks that need click through rate calculation. Default value @@ -50,7 +51,14 @@ class WindowedClickThroughRate( >>> metric.update(torch.tensor([0, 1, 0, 1, 1, 1, 1, 1])) >>> metric.update(torch.tensor([0, 1, 0, 1, 0, 0, 0, 1])) >>> metric.compute() - tensor([0.5625]) + (tensor([0.5417], dtype=torch.float64), tensor([0.5625], dtype=torch.float64)) + + >>> metric = WindowedClickThroughRate(max_num_updates=2, enable_lifetime=False) + >>> metric.update(torch.tensor([0, 1, 0, 1, 1, 0, 0, 1])) + >>> metric.update(torch.tensor([0, 1, 0, 1, 1, 1, 1, 1])) + >>> metric.update(torch.tensor([0, 1, 0, 1, 0, 0, 0, 1])) + >>> metric.compute() + tensor([0.5625], dtype=torch.float64) """ @@ -76,21 +84,22 @@ def __init__( self.next_inserted = 0 self.enable_lifetime = enable_lifetime self._add_state("total_updates", 0) + dtype = largest_float(device) if self.enable_lifetime: self._add_state( "click_total", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "weight_total", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "windowed_click_total", torch.zeros( self.num_tasks, self.max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ), ) @@ -99,7 +108,7 @@ def __init__( torch.zeros( self.num_tasks, self.max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ), ) @@ -141,10 +150,11 @@ def compute( ``compute()`` is called, return tensor(0.0). """ if self.total_updates == 0: + naught = torch.empty(0, device=self.device) if self.enable_lifetime: - return torch.empty(0), torch.empty(0) + return naught, naught else: - return torch.empty(0) + return naught # For the case that window has been filled more than once if self.total_updates >= self.max_num_updates: @@ -187,16 +197,17 @@ def merge_state( cur_click_total = self.windowed_click_total cur_weight_total = self.windowed_weight_total + dtype = largest_float(self.device) self.windowed_click_total = torch.zeros( self.num_tasks, merge_max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ) self.windowed_weight_total = torch.zeros( self.num_tasks, merge_max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ) diff --git a/torcheval/metrics/window/mean_squared_error.py b/torcheval/metrics/window/mean_squared_error.py index 7573839..ef2eba6 100644 --- a/torcheval/metrics/window/mean_squared_error.py +++ b/torcheval/metrics/window/mean_squared_error.py @@ -169,10 +169,11 @@ def compute( Empty tensor is returned if no calls to ``update()`` are made before ``compute()`` is called. """ if self.total_updates == 0: + naught = torch.empty(0, device=self.device) if self.enable_lifetime: - return torch.empty(0), torch.empty(0) + return naught, naught else: - return torch.empty(0) + return naught windowed_sum_squared_error = self.windowed_sum_squared_error.sum(dim=1) windowed_sum_weight = self.windowed_sum_weight.sum(dim=1) diff --git a/torcheval/metrics/window/normalized_entropy.py b/torcheval/metrics/window/normalized_entropy.py index a701727..c550b2d 100644 --- a/torcheval/metrics/window/normalized_entropy.py +++ b/torcheval/metrics/window/normalized_entropy.py @@ -18,6 +18,7 @@ _binary_normalized_entropy_update, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TWindowedNormalizedEntropy = TypeVar("TWindowedNormalizedEntropy") @@ -104,18 +105,19 @@ def __init__( self.enable_lifetime = enable_lifetime self._add_state("total_updates", 0) + dtype = largest_float(device) if self.enable_lifetime: self._add_state( "total_entropy", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "num_examples", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "num_positive", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( @@ -123,7 +125,7 @@ def __init__( torch.zeros( self.num_tasks, self.max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ), ) @@ -132,7 +134,7 @@ def __init__( torch.zeros( self.num_tasks, self.max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ), ) @@ -141,7 +143,7 @@ def __init__( torch.zeros( self.num_tasks, self.max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ), ) @@ -196,10 +198,11 @@ def compute( The return tensors is binary normalized entropy for each task (num_tasks,). """ if self.total_updates == 0: + naught = torch.empty(0, device=self.device) if self.enable_lifetime: - return torch.empty(0), torch.empty(0) + return naught, naught else: - return torch.empty(0) + return naught # For the case that window has been filled more than once if self.total_updates >= self.max_num_updates: @@ -251,22 +254,23 @@ def merge_state( cur_total_entropy = self.windowed_total_entropy cur_num_examples = self.windowed_num_examples cur_num_positive = self.windowed_num_positive + dtype = largest_float(self.device) self.windowed_total_entropy = torch.zeros( self.num_tasks, merge_max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ) self.windowed_num_examples = torch.zeros( self.num_tasks, merge_max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ) self.windowed_num_positive = torch.zeros( self.num_tasks, merge_max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ) diff --git a/torcheval/metrics/window/weighted_calibration.py b/torcheval/metrics/window/weighted_calibration.py index e06a07e..8350a5f 100644 --- a/torcheval/metrics/window/weighted_calibration.py +++ b/torcheval/metrics/window/weighted_calibration.py @@ -17,6 +17,7 @@ _weighted_calibration_update, ) from torcheval.metrics.metric import Metric +from torcheval.utils.device import largest_float TWindowedWeightedCalibration = TypeVar("TWindowedWeightedCalibration") @@ -85,21 +86,22 @@ def __init__( self.enable_lifetime = enable_lifetime self.next_inserted = 0 self._add_state("total_updates", 0) + dtype = largest_float(device) if self.enable_lifetime: self._add_state( "weighted_input_sum", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "weighted_target_sum", - torch.zeros(self.num_tasks, dtype=torch.float64, device=self.device), + torch.zeros(self.num_tasks, dtype=dtype, device=self.device), ) self._add_state( "windowed_weighted_input_sum", torch.zeros( self.num_tasks, self.max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ), ) @@ -108,7 +110,7 @@ def __init__( torch.zeros( self.num_tasks, self.max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ), ) @@ -156,17 +158,19 @@ def compute( Tensor: The return value of weighted calibration for each task (num_tasks,). """ if self.total_updates == 0: + naught = torch.empty(0, device=self.device) if self.enable_lifetime: - return torch.empty(0), torch.empty(0) + return naught, naught else: - return torch.empty(0) + return naught # for the case the winodw has been filled more than once + dtype = largest_float(self.device) if self.total_updates >= self.max_num_updates: windowed_weighted_calibration = self.windowed_weighted_input_sum.sum( dim=-1 ) / torch.clamp( self.windowed_weighted_target_sum.sum(dim=-1), - min=torch.finfo(torch.float64).eps, + min=torch.finfo(dtype).eps, ) else: # for the situation when window array hasn't been filled up @@ -174,11 +178,11 @@ def compute( :, : self.next_inserted ].sum(dim=-1) / torch.clamp( self.windowed_weighted_target_sum[:, : self.next_inserted].sum(dim=-1), - min=torch.finfo(torch.float64).eps, + min=torch.finfo(dtype).eps, ) if self.enable_lifetime: self.weighted_target_sum = torch.clamp( - self.weighted_target_sum, min=torch.finfo(torch.float64).eps + self.weighted_target_sum, min=torch.finfo(dtype).eps ) weighted_calibration = self.weighted_input_sum / self.weighted_target_sum return weighted_calibration, windowed_weighted_calibration @@ -201,16 +205,17 @@ def merge_state( cur_windowed_weighted_input_sum = self.windowed_weighted_input_sum cur_windowed_weighted_target_sum = self.windowed_weighted_target_sum idx = min(self.total_updates, self.max_num_updates) + dtype = largest_float(self.device) self.windowed_weighted_input_sum = torch.zeros( self.num_tasks, merge_max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ) self.windowed_weighted_target_sum = torch.zeros( self.num_tasks, merge_max_num_updates, - dtype=torch.float64, + dtype=dtype, device=self.device, ) self.windowed_weighted_input_sum[:, :idx] = cur_windowed_weighted_input_sum[ diff --git a/torcheval/utils/device.py b/torcheval/utils/device.py new file mode 100644 index 0000000..86a97b5 --- /dev/null +++ b/torcheval/utils/device.py @@ -0,0 +1,29 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-strict + +import functools +from typing import Union + +import torch + + +@functools.lru_cache() +def largest_float(device: Union[torch.device, str, None]) -> torch.dtype: + """Determines whether the largest representable floating-point type on + a given device is 64-bit or 32-bit. + + Args: + device (Union[torch.device, str, None]) + + Returns: + torch.dtype: either torch.float64 or torch.float32""" + try: + torch.zeros(1, dtype=torch.float64, device=device) + return torch.float64 + except TypeError: + return torch.float32