Merge pull request #514 from scikit-learn-contrib/511-modulenotfounde…

…rror-no-module-named-mapieconformity_scoresresidual_conformity_scores

mapie/conformity_scores/conformity_scores.py

@@ -0,0 +1,165 @@
+from abc import ABCMeta, abstractmethod
+
+from sklearn.utils import deprecated
+
+from mapie.conformity_scores.regression import BaseConformityScore
+from mapie._machine_precision import EPSILON
+from mapie._typing import NDArray
+
+
+@deprecated(
+ "WARNING: Deprecated path to import ConformityScore. "
+ "Please prefer the new path: "
+ "[from mapie.conformity_scores import BaseRegressionScore]."
+)
+class ConformityScore(BaseConformityScore, metaclass=ABCMeta):
+ """
+ Base conformity score class for regression task.
+
+ This class should not be used directly. Use derived classes instead.
+
+ Parameters
+ ----------
+ sym: bool
+ Whether to consider the conformity score as symmetrical or not.
+
+ consistency_check: bool, optional
+ Whether to check the consistency between the methods
+ ``get_estimation_distribution`` and ``get_conformity_scores``.
+ If ``True``, the following equality must be verified:
+ ``self.get_estimation_distribution(
+ y_pred, self.get_conformity_scores(y, y_pred, **kwargs), **kwargs
+ ) == y``
+
+ By default ``True``.
+
+ eps: float, optional
+ Threshold to consider when checking the consistency between
+ ``get_estimation_distribution`` and ``get_conformity_scores``.
+ It should be specified if ``consistency_check==True``.
+
+ By default, it is defined by the default precision.
+ """
+
+ def __init__(
+ self,
+ sym: bool,
+ consistency_check: bool = True,
+ eps: float = float(EPSILON),
+ ):
+ super().__init__()
+ self.sym = sym
+ self.consistency_check = consistency_check
+ self.eps = eps
+
+ @abstractmethod
+ def get_signed_conformity_scores(
+ self,
+ y: NDArray,
+ y_pred: NDArray,
+ **kwargs
+ ) -> NDArray:
+ """
+ Placeholder for ``get_conformity_scores``.
+ Subclasses should implement this method!
+
+ Compute the sample conformity scores given the predicted and
+ observed targets.
+
+ Parameters
+ ----------
+ y: NDArray of shape (n_samples,)
+ Observed target values.
+
+ y_pred: NDArray of shape (n_samples,)
+ Predicted target values.
+
+ Returns
+ -------
+ NDArray of shape (n_samples,)
+ Signed conformity scores.
+ """
+
+ @abstractmethod
+ def get_conformity_scores(
+ self,
+ y: NDArray,
+ y_pred: NDArray,
+ **kwargs
+ ) -> NDArray:
+ """
+ Placeholder for ``get_conformity_scores``.
+ Subclasses should implement this method!
+
+ Compute the sample conformity scores given the predicted and
+ observed targets.
+
+ Parameters
+ ----------
+ y: NDArray of shape (n_samples,)
+ Observed target values.
+
+ y_pred: NDArray of shape (n_samples,)
+ Predicted target values.
+
+ Returns
+ -------
+ NDArray of shape (n_samples,)
+ Conformity scores.
+ """
+
+ @abstractmethod
+ def get_estimation_distribution(
+ self,
+ y_pred: NDArray,
+ conformity_scores: NDArray,
+ **kwargs
+ ) -> NDArray:
+ """
+ Placeholder for ``get_estimation_distribution``.
+ Subclasses should implement this method!
+
+ Compute samples of the estimation distribution given the predicted
+ targets and the conformity scores.
+
+ Parameters
+ ----------
+ y_pred: NDArray of shape (n_samples,)
+ Predicted target values.
+
+ conformity_scores: NDArray of shape (n_samples,)
+ Conformity scores.
+
+ Returns
+ -------
+ NDArray of shape (n_samples,)
+ Observed values.
+ """
+
+ @abstractmethod
+ def predict_set(
+ self,
+ X: NDArray,
+ alpha_np: NDArray,
+ **kwargs
+ ):
+ """
+ Compute the prediction sets on new samples based on the uncertainty of
+ the target confidence set.
+
+ Parameters:
+ -----------
+ X: NDArray of shape (n_samples,)
+ The input data or samples for prediction.
+
+ alpha_np: NDArray of shape (n_alpha, )
+ Represents the uncertainty of the confidence set to produce.
+
+ **kwargs: dict
+ Additional keyword arguments.
+
+ Returns:
+ --------
+ The output structure depend on the subclass.
+ The prediction sets for each sample and each alpha level.
+ """

mapie/conformity_scores/residual_conformity_scores.py

@@ -0,0 +1,34 @@
+from sklearn.utils import deprecated
+
+from .bounds import (
+ AbsoluteConformityScore as NewAbsoluteConformityScore,
+ GammaConformityScore as NewGammaConformityScore,
+ ResidualNormalisedScore as NewResidualNormalisedScore
+)
+
+
+@deprecated(
+ "WARNING: Deprecated path to import AbsoluteConformityScore. "
+ "Please prefer the new path: "
+ "[from mapie.conformity_scores.bounds import AbsoluteConformityScore]."
+)
+class AbsoluteConformityScore(NewAbsoluteConformityScore):
+ pass
+
+
+@deprecated(
+ "WARNING: Deprecated path to import GammaConformityScore. "
+ "Please prefer the new path: "
+ "[from mapie.conformity_scores.bounds import GammaConformityScore]."
+)
+class GammaConformityScore(NewGammaConformityScore):
+ pass
+
+
+@deprecated(
+ "WARNING: Deprecated path to import ResidualNormalisedScore. "
+ "Please prefer the new path: "
+ "[from mapie.conformity_scores.bounds import ResidualNormalisedScore]."
+)
+class ResidualNormalisedScore(NewResidualNormalisedScore):
+ pass

mapie/conformity_scores/utils_classification_conformity_scores.py

@@ -0,0 +1,14 @@
+from sklearn.utils import deprecated
+
+from mapie.conformity_scores.sets.utils import (
+ get_true_label_position as get_true_label_position_new_path,
+)
+
+
+@deprecated(
+ "WARNING: Deprecated path to import get_true_label_position. "
+ "Please prefer the new path: "
+ "[from mapie.conformity_scores.sets.utils import get_true_label_position]."
+)
+def get_true_label_position(*args, **kwargs):
+ return get_true_label_position_new_path(*args, **kwargs)

mapie/estimator/estimator.py

@@ -0,0 +1,12 @@
+from sklearn.utils import deprecated
+
+from mapie.estimator.regressor import EnsembleRegressor as NewEnsembleRegressor
+
+
+@deprecated(
+ "WARNING: Deprecated path to import EnsembleRegressor. "
+ "Please prefer the new path: "
+ "[from mapie.estimator.regressor import EnsembleRegressor]."
+)
+class EnsembleRegressor(NewEnsembleRegressor):
+ pass

mapie/tests/test_common.py

@@ -11,13 +11,14 @@
 from sklearn.utils.estimator_checks import parametrize_with_checks
 from sklearn.utils.validation import check_is_fitted
 
+from mapie._typing import ArrayLike, NDArray
 from mapie.classification import MapieClassifier
 from mapie.regression import MapieQuantileRegressor, MapieRegressor
 
 X_toy = np.arange(18).reshape(-1, 1)
 y_toy = np.array(
  [0, 0, 1, 0, 1, 2, 1, 2, 2, 0, 0, 1, 0, 1, 2, 1, 2, 2]
- )
+)
 
 
 def MapieSimpleEstimators() -> List[BaseEstimator]:
@@ -195,3 +196,121 @@ def test_sklearn_compatible_estimator(
 ) -> None:
  """Check compatibility with sklearn, using sklearn estimator checks API."""
  check(estimator)
+
+
+def test_warning_when_import_from_gamma_conformity_score():
+ """Check that a DepreciationWarning is raised when importing from
+ mapie.conformity_scores.residual_conformity_scores"""
+
+ with pytest.warns(
+ FutureWarning, match=r".*WARNING: Deprecated path to import.*"
+ ):
+ from mapie.conformity_scores.residual_conformity_scores import (
+ GammaConformityScore
+ )
+ GammaConformityScore()
+
+
+def test_warning_when_import_from_absolute_conformity_score():
+ """Check that a DepreciationWarning is raised when importing from
+ mapie.conformity_scores.residual_conformity_scores"""
+
+ with pytest.warns(
+ FutureWarning, match=r".*WARNING: Deprecated path to import.*"
+ ):
+ from mapie.conformity_scores.residual_conformity_scores import (
+ AbsoluteConformityScore
+ )
+ AbsoluteConformityScore()
+
+
+def test_warning_when_import_from_residual_conformity_score():
+ """Check that a DepreciationWarning is raised when importing from
+ mapie.conformity_scores.residual_conformity_scores"""
+
+ with pytest.warns(
+ FutureWarning, match=r".*WARNING: Deprecated path to import.*"
+ ):
+ from mapie.conformity_scores.residual_conformity_scores import (
+ ResidualNormalisedScore
+ )
+ ResidualNormalisedScore()
+
+
+def test_warning_when_import_from_conformity_scores():
+ """Check that a DepreciationWarning is raised when importing from
+ mapie.conformity_scores.conformity_score"""
+
+ with pytest.warns(
+ FutureWarning, match=r".*WARNING: Deprecated path to import.*"
+ ):
+ from mapie.conformity_scores.conformity_scores import (
+ ConformityScore
+ )
+
+ class DummyConformityScore(ConformityScore):
+ def __init__(self) -> None:
+ super().__init__(sym=True, consistency_check=True)
+
+ def get_signed_conformity_scores(
+ self, y: ArrayLike, y_pred: ArrayLike, **kwargs
+ ) -> NDArray:
+ return np.array([])
+
+ def get_estimation_distribution(
+ self, y_pred: ArrayLike, conformity_scores: ArrayLike, **kwargs
+ ) -> NDArray:
+ """
+ A positive constant is added to the sum between predictions and
+ conformity scores to make the estimated distribution
+ inconsistent with the conformity score.
+ """
+ return np.array([])
+
+ def get_conformity_scores(
+ self, y: ArrayLike, y_pred: ArrayLike, **kwargs
+ ) -> NDArray:
+ return np.array([])
+
+ def predict_set(
+ self, X: NDArray, alpha_np: NDArray, **kwargs
+ ) -> NDArray:
+ return np.array([])
+
+ dcs = DummyConformityScore()
+ dcs.get_signed_conformity_scores(y_toy, y_toy)
+ dcs.get_estimation_distribution(y_toy, y_toy)
+ dcs.get_conformity_scores(y_toy, y_toy)
+ dcs.predict_set(y_toy, 0.5)
+
+
+def test_warning_when_import_from_old_get_true_label_position():
+ """Check that a DepreciationWarning is raised when importing from
+ mapie.conformity_scores.residual_conformity_scores"""
+
+ with pytest.warns(
+ FutureWarning, match=r".*WARNING: Deprecated path to import.*"
+ ):
+ from mapie.conformity_scores.utils_classification_conformity_scores\
+ import get_true_label_position
+ get_true_label_position(np.array([[0.1, 0.2, 0.7]]), np.array([2]))
+
+
+def test_warning_when_import_from_estimator():
+ """Check that a DepreciationWarning is raised when importing from
+ mapie.estimator.estimator"""
+
+ with pytest.warns(
+ FutureWarning, match=r".*WARNING: Deprecated path to import.*"
+ ):
+ from mapie.estimator.estimator import EnsembleRegressor
+ EnsembleRegressor(
+ estimator=LinearRegression(),
+ method="naive",
+ cv=3,
+ agg_function="mean",
+ n_jobs=1,
+ random_state=0,
+ test_size=0.2,
+ verbose=0,
+ )