fixed bug in holdout data summary metrics per horizon (#377)

oracle · Oct 19, 2023 · 6e33aef · 6e33aef
2 parents 39c3b9a + 3c220c3
commit 6e33aef
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 8 deletions.
diff --git a/ads/opctl/operator/lowcode/forecast/const.py b/ads/opctl/operator/lowcode/forecast/const.py
@@ -27,14 +27,14 @@ class SupportedMetrics(str, metaclass=ExtendedEnumMeta):
     WMAPE = "wMAPE"
     R2 = "r2"
     EXPLAINED_VARIANCE = "Explained Variance"
-    MEAN_MAPE = "Mean MAPE" 
+    MEAN_MAPE = "Mean MAPE"
     MEAN_RMSE = "Mean RMSE"
     MEAN_MSE = "Mean MSE"
     MEAN_SMAPE = "Mean sMAPE"
     MEAN_WMAPE = "Mean wMAPE"
     MEAN_R2 = "Mean r2"
     MEAN_EXPLAINED_VARIANCE = "Mean Explained Variance"
-    MEDIAN_MAPE = "Median MAPE" 
+    MEDIAN_MAPE = "Median MAPE"
     MEDIAN_RMSE = "Median RMSE"
     MEDIAN_MSE = "Median MSE"
     MEDIAN_SMAPE = "Median sMAPE"
@@ -54,4 +54,5 @@ class SupportedMetrics(str, metaclass=ExtendedEnumMeta):
 }
 
 MAX_COLUMNS_AUTOMLX = 15
-DEFAULT_TRIALS = 10
+DEFAULT_TRIALS = 10
+SUMMARY_METRICS_HORIZON_LIMIT = 10
diff --git a/ads/opctl/operator/lowcode/forecast/model/base_model.py b/ads/opctl/operator/lowcode/forecast/model/base_model.py
@@ -18,7 +18,7 @@
 from ads.opctl import logger
 
 from .. import utils
-from ..const import SupportedModels, SupportedMetrics
+from ..const import SupportedModels, SupportedMetrics, SUMMARY_METRICS_HORIZON_LIMIT
 from ..operator_config import ForecastOperatorConfig, ForecastOperatorSpec
 from .transformations import Transformations
 
@@ -80,7 +80,6 @@ def generate_report(self):
             ci_col_names,
         ) = self._generate_report()
 
-
         report_sections = []
         title_text = dp.Text("# Forecast Report")
 
@@ -342,12 +341,15 @@ def _test_evaluate_metrics(
         """Calculates Mean sMAPE, Median sMAPE, Mean MAPE, Median MAPE, Mean wMAPE, Median wMAPE values for each horizon
         if horizon <= 10."""
         target_columns_in_output = set(target_columns).intersection(data.columns)
-        if len(data["ds"]) <= 10 and len(outputs) == len(target_columns_in_output):
+        if self.spec.horizon.periods <= SUMMARY_METRICS_HORIZON_LIMIT and len(
+            outputs
+        ) == len(target_columns_in_output):
             metrics_per_horizon = utils._build_metrics_per_horizon(
                 data=data,
                 outputs=outputs,
                 target_columns=target_columns,
                 target_col=target_col,
+                horizon_periods=self.spec.horizon.periods,
             )
 
             summary_metrics = summary_metrics.append(metrics_per_horizon)

diff --git a/ads/opctl/operator/lowcode/forecast/utils.py b/ads/opctl/operator/lowcode/forecast/utils.py
@@ -50,6 +50,7 @@ def _build_metrics_per_horizon(
     outputs: pd.DataFrame,
     target_columns: List[str],
     target_col: str,
+    horizon_periods: int,
 ) -> pd.DataFrame:
     """
     Calculates Mean sMAPE, Median sMAPE, Mean MAPE, Median MAPE, Mean wMAPE, Median wMAPE for each horizon
@@ -64,15 +65,18 @@ def _build_metrics_per_horizon(
             List of target category columns
     target_col: str
             Target column name (yhat)
+    horizon_periods: int
+            Horizon Periods
 
     Returns
     --------
     Pandas Dataframe
         Dataframe with Mean sMAPE, Median sMAPE, Mean MAPE, Median MAPE, Mean wMAPE, Median wMAPE values for each horizon
     """
     actuals_df = data[target_columns]
-    forecasts_df = pd.concat([df[target_col] for df in outputs], axis=1)
-
+    forecasts_df = pd.concat(
+        [df[target_col].iloc[-horizon_periods:] for df in outputs], axis=1
+    )
     totals = actuals_df.sum()
     wmape_weights = np.array((totals / totals.sum()).values)