From f6606be027d32e6031478263724a80824ae598b8 Mon Sep 17 00:00:00 2001 From: Alex Golts Date: Mon, 4 Dec 2023 11:15:46 +0200 Subject: [PATCH] pearson correlation metric (#338) * pearson correlation metric * formatting * add optional mask to pearsonr metric * formatting * fixed random vector --- fuse/eval/examples/examples_stats.py | 55 +++++++++++++++++++ fuse/eval/metrics/libs/stat.py | 46 ++++++++++++++++ fuse/eval/metrics/stat/metrics_stat_common.py | 16 +++++- fuse/eval/tests/test_eval.py | 6 ++ 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 fuse/eval/examples/examples_stats.py create mode 100644 fuse/eval/metrics/libs/stat.py diff --git a/fuse/eval/examples/examples_stats.py b/fuse/eval/examples/examples_stats.py new file mode 100644 index 00000000..782d18fd --- /dev/null +++ b/fuse/eval/examples/examples_stats.py @@ -0,0 +1,55 @@ +""" +(C) Copyright 2021 IBM Corp. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Created on Nov 30, 2023 + +""" + +from fuse.eval.metrics.stat.metrics_stat_common import MetricPearsonCorrelation +import numpy as np +import pandas as pd +from collections import OrderedDict +from fuse.eval.evaluator import EvaluatorDefault + + +def example_pearson_correlation() -> float: + """ + Pearson correlation coefficient + """ + + # define data + sz = 1000 + data = { + "id": range(sz), + } + np.random.seed(0) + rand_vec = np.random.randn((sz)) + data["x1"] = 100 * np.ones((sz)) + 10 * rand_vec + data["x2"] = -10 * np.ones((sz)) + 3 * rand_vec + + data_df = pd.DataFrame(data) + + # list of metrics + metrics = OrderedDict( + [ + ("pearsonr", MetricPearsonCorrelation(pred="x1", target="x2")), + ] + ) + + # read files + evaluator = EvaluatorDefault() + res = evaluator.eval(ids=None, data=data_df, metrics=metrics) + + return res diff --git a/fuse/eval/metrics/libs/stat.py b/fuse/eval/metrics/libs/stat.py new file mode 100644 index 00000000..8054849c --- /dev/null +++ b/fuse/eval/metrics/libs/stat.py @@ -0,0 +1,46 @@ +import numpy as np +from typing import Sequence, Union + + +class Stat: + """ + Statistical metrics + """ + + @staticmethod + def pearson_correlation( + pred: Union[np.ndarray, Sequence], + target: Union[np.ndarray, Sequence], + mask: Union[np.ndarray, Sequence, None] = None, + ) -> float: + """ + Pearson correlation coefficient measuring the linear relationship between two datasets/vectors. + :param pred: prediction values + :param target: target values + :param mask: optional boolean mask. if it is provided, the metric will be applied only to the masked samples + """ + if isinstance(pred, Sequence): + pred = np.array(pred) + if isinstance(target, Sequence): + target = np.array(target) + if isinstance(mask, Sequence): + mask = np.array(mask).astype("bool") + if mask is not None: + pred = pred[mask] + target = target[mask] + + pred = pred.squeeze() + target = target.squeeze() + if len(pred.shape) > 1 or len(target.shape) > 1: + raise ValueError( + f"expected 1D vectors. got pred shape: {pred.shape}, target shape: {target.shape}" + ) + + mean_pred = np.mean(pred) + mean_target = np.mean(target) + + r = np.sum((pred - mean_pred) * (target - mean_target)) / np.sqrt( + np.sum((pred - mean_pred) ** 2) * np.sum((target - mean_target) ** 2) + ) + + return r diff --git a/fuse/eval/metrics/stat/metrics_stat_common.py b/fuse/eval/metrics/stat/metrics_stat_common.py index 7cf3eaaa..15d47550 100644 --- a/fuse/eval/metrics/stat/metrics_stat_common.py +++ b/fuse/eval/metrics/stat/metrics_stat_common.py @@ -1,6 +1,7 @@ from typing import Any, Dict, Hashable, Optional, Sequence from collections import Counter -from fuse.eval.metrics.metrics_common import MetricWithCollectorBase +from fuse.eval.metrics.metrics_common import MetricDefault, MetricWithCollectorBase +from fuse.eval.metrics.libs.stat import Stat class MetricUniqueValues(MetricWithCollectorBase): @@ -19,3 +20,16 @@ def eval( counter = Counter(values) return list(counter.items()) + + +class MetricPearsonCorrelation(MetricDefault): + def __init__( + self, pred: str, target: str, mask: Optional[str] = None, **kwargs: dict + ) -> None: + super().__init__( + pred=pred, + target=target, + mask=mask, + metric_func=Stat.pearson_correlation, + **kwargs + ) diff --git a/fuse/eval/tests/test_eval.py b/fuse/eval/tests/test_eval.py index 90621ff2..b54492a9 100644 --- a/fuse/eval/tests/test_eval.py +++ b/fuse/eval/tests/test_eval.py @@ -50,6 +50,8 @@ from fuse.eval.examples.examples_seq_gen import example_seq_gen_0, example_seq_gen_1 +from fuse.eval.examples.examples_stats import example_pearson_correlation + class TestEval(unittest.TestCase): def test_eval_example_0(self) -> None: @@ -232,6 +234,10 @@ def test_eval_example_seq_gen_1(self) -> None: results = example_seq_gen_1(seed=1234) self.assertAlmostEqual(results["metrics.perplexity"], 162.87, places=2) + def test_pearson_correlation(self) -> None: + res = example_pearson_correlation() + self.assertAlmostEqual(res["metrics.pearsonr"], 1.0, places=2) + if __name__ == "__main__": unittest.main()