Price taker model for DISPATCHES, Rehashed

idaes/apps/grid_integration/multiperiod/price_taker_model.py

@@ -25,16 +25,7 @@
 import pandas as pd
 import numpy as np
 
-have_skl = True
-have_kn = True
-try:
-    from sklearn.cluster import KMeans
-except ImportError:
-    have_skl = False
-try:
-    from kneed import KneeLocator
-except ImportError:
-    have_kn = False
+from scipy.cluster.vq import kmeans, kmeans2, vq, whiten
 
 import matplotlib.pyplot as plt
 
@@ -50,7 +41,6 @@
     NonNegativeReals,
     Expression,
     maximize,
-    Param,
 )
 
 from idaes.apps.grid_integration import MultiPeriodModel
@@ -77,10 +67,9 @@ def seed(self):
 
     @seed.setter
     def seed(self, value):
-        try:
-            self._seed = value
-        except TypeError:
+        if not isinstance(value, int):
             raise TypeError(f"seed must be an integer, but {value} is not an integer")
+        self._seed = value
 
     @property
     def horizon_length(self):
@@ -91,11 +80,30 @@ def horizon_length(self, value):
         if value <= 0:
             raise ValueError(f"horizon_length must be > 0, but {value} is provided.")
         if not isinstance(value, int):
-            raise ValueError(
+            raise TypeError(
                 f"horizon_length must be an integer, but {value} is not an integer"
             )
         self._horizon_length = value
 
+    @staticmethod
+    def compute_sse(data, centroids, idx):
+        """
+        Function used to compute the inertia (sum of square errors) for k clusters.
+
+        Args:
+            data:      Columnar data for a given LMP signal
+            centroids: Array of k centroids
+            idx:       Index for data
+
+        Returns:
+            inertia: Sum of square errors for k clusters
+        """
+        inertia = 0
+        for i, centroid in enumerate(centroids):
+            cluster_points = data[idx == i]
+            inertia += np.sum((cluster_points - centroid) ** 2)
+        return inertia
+
     def generate_daily_data(self, raw_data):
         """
         Function used to generate the daily data in a usable format
@@ -171,20 +179,23 @@ def get_optimal_n_clusters(
 
         np.random.seed(self._seed)
 
-        if not (have_skl and have_kn):
-            raise ImportError(
-                f"Optimal cluster feature requires optional imports 'scikit-learn' and 'kneed'."
-            )
-        # Compute the inertia (SSE) for k clusters
+        whitened_daily_data = whiten(daily_data.transpose())
+
         for k in k_values:
-            kmeans = KMeans(n_clusters=k).fit(daily_data.transpose())
-            inertia_values.append(kmeans.inertia_)
+            centroids, _ = kmeans(whitened_daily_data, k)
+            idx, _ = vq(whitened_daily_data, centroids)
 
-        # Identify the "elbow point"
-        elbow_point = KneeLocator(
-            k_values, inertia_values, curve="convex", direction="decreasing"
-        )
-        n_clusters = elbow_point.knee
+            # Compute the inertia (SSE) for k clusters
+            inertia = self.compute_sse(whitened_daily_data, centroids, idx)
+            inertia_values.append(inertia)
+
+        # Calculate the second derivative
+        first_deriv = np.diff(inertia_values)
+        second_deriv = np.diff(first_deriv)
+
+        # Determine the optimal number of clusters
+        # The +2 accounts for the dimension being reduced twice by derivatives
+        n_clusters = np.argmin(second_deriv) + 2
 
         if n_clusters is None:
             raise ValueError(
@@ -272,15 +283,10 @@ def cluster_lmp_data(self, raw_data, n_clusters):
 
         # reconfiguring raw data
         daily_data = self.generate_daily_data(raw_data)
+        whitened_daily_data = whiten(daily_data.transpose())
 
         # KMeans clustering with the optimal number of clusters
-        if not have_skl:
-            raise ImportError(
-                f"Clustering feature requires optional import 'scikit-learn'."
-            )
-        kmeans = KMeans(n_clusters=n_clusters).fit(daily_data.transpose())
-        centroids = kmeans.cluster_centers_
-        labels = kmeans.labels_
+        centroids, labels = kmeans2(whitened_daily_data, n_clusters)
 
         # Set any centroid values that are < 1e-4 to 0 to avoid noise
         centroids = centroids * (abs(centroids) >= 1e-4)

idaes/apps/grid_integration/multiperiod/tests/test_price_taker.py

@@ -36,18 +36,6 @@
 
 from idaes.apps.grid_integration.multiperiod.price_taker_model import PriceTakerModel
 
-have_skl = True
-have_kn = True
-try:
-    import sklearn as skl
-    from sklearn.cluster import KMeans
-except:
-    have_skl = False
-try:
-    from kneed import KneeLocator
-except:
-    have_kn = False
-
 
 @pytest.fixture
 def excel_data():
@@ -57,14 +45,6 @@ def excel_data():
     return data
 
 
-@pytest.fixture
-def sample_data():
-    DATA_DIR = Path(__file__).parent
-    file_path = DATA_DIR / "sample_data.csv"
-    data = pd.read_csv(file_path)
-    return data
-
-
 @pytest.mark.unit
 def test_seed():
     m = PriceTakerModel()
@@ -75,7 +55,7 @@ def test_seed():
 def test_seed_value():
     value = "fifty"
     with pytest.raises(
-        ValueError,
+        TypeError,
         match=("seed must be an integer, but fifty is not an integer"),
     ):
 
@@ -94,40 +74,16 @@ def test_daily_data_size(excel_data):
     assert len(daily_data) == m.horizon_length
 
 
-@pytest.mark.skipif(
-    have_skl * have_kn < 1,
-    reason="optional packages 'scikit-learn' and 'kneed' not installed",
-)
 @pytest.mark.unit
-def test_skl_version(excel_data):
-    skl_version = skl.__version__
-    major, minor, *_ = map(int, skl_version.split("."))
-
-    # Checks that the version is 2.0+ or greater than 1.1
-    # Only versions at or above 1.1 support python 3.12
-    assert major > 1 or (major == 1 and minor >= 1)
-
-
-@pytest.mark.skipif(
-    have_skl * have_kn < 1,
-    reason="optional packages 'scikit-learn' and 'kneed' not installed",
-)
-@pytest.mark.unit
-def test_determine_optimal_num_clusters(sample_data):
-    # Uses the smaller sample_data dataset so that the solution
-    # reliably converges to 9 clusters, instead of a range of clusters
+def test_determine_optimal_num_clusters(excel_data):
     m = PriceTakerModel()
 
-    daily_data = m.generate_daily_data(sample_data["BaseCaseTax"])
-    n_clusters, inertia_values = m.get_optimal_n_clusters(daily_data, kmax=15)
+    daily_data = m.generate_daily_data(excel_data["BaseCaseTax"])
+    n_clusters, inertia_values = m.get_optimal_n_clusters(daily_data, kmax=30)
 
-    assert n_clusters == 9
+    assert n_clusters == 11
 
 
-@pytest.mark.skipif(
-    have_skl * have_kn < 1,
-    reason="optional packages 'scikit-learn' and 'kneed' not installed",
-)
 @pytest.mark.unit
 def test_generate_elbow_plot(excel_data):
     m = PriceTakerModel()
@@ -145,10 +101,6 @@ def test_generate_elbow_plot(excel_data):
     plt.close("all")
 
 
-@pytest.mark.skipif(
-    not (have_skl and have_kn),
-    reason="optional packages 'scikit-learn' and 'kneed' not installed",
-)
 @pytest.mark.unit
 def test_cluster_lmp_data(excel_data):
     # This function gets within both if statement expression in the
@@ -169,10 +121,6 @@ def test_cluster_lmp_data(excel_data):
     assert len(lmp_data) == n_clusters
 
 
-@pytest.mark.skipif(
-    not (have_skl and have_kn),
-    reason="optional packages 'scikit-learn' and 'kneed' not installed",
-)
 @pytest.mark.unit
 def test_init_logger_messages_clusters(excel_data, caplog):
     with caplog.at_level(idaeslog.WARNING):
@@ -201,7 +149,7 @@ def test_init_logger_message2(excel_data, caplog):
     value = 12.34
     m = PriceTakerModel()
     with pytest.raises(
-        ValueError,
+        TypeError,
         match=("horizon_length must be an integer, but 12.34 is not an integer"),
     ):
         m.horizon_length = value
@@ -213,7 +161,7 @@ def test_init_logger_message3(excel_data, caplog):
     value = 12.34
     m = PriceTakerModel()
     with pytest.raises(
-        ValueError,
+        TypeError,
         match=("seed must be an integer, but 12.34 is not an integer"),
     ):
         m.seed = value
@@ -279,10 +227,6 @@ def test_min_down_time_logger_message2(excel_data):
         m.add_startup_shutdown(des, oper, build_bin_var, up_time[0], down_time[1])
 
 
-@pytest.mark.skipif(
-    not (have_skl and have_kn),
-    reason="optional package 'scikit-learn' not installed",
-)
 @pytest.mark.unit
 def test_init_logger_messages_clusters_min_up_down_time(excel_data, caplog):
     # Test Not Implemented Error (Rep. Days used for su/sd code)
@@ -294,7 +238,7 @@ def test_init_logger_messages_clusters_min_up_down_time(excel_data, caplog):
     m.append_lmp_data(
         file_path=file_path,
         column_name="BaseCaseTax",
-        n_clusters=5,
+        n_clusters=2,
     )
 
     m.sofc_design = DesignModel(
@@ -401,25 +345,21 @@ def test_optimal_clusters_kmax_logger_message2(excel_data):
         m.cluster_lmp_data(excel_data, n_clusters)
 
 
-@pytest.mark.skipif(
-    have_skl and have_kn,
-    reason="optional packages 'scikit-learn' and 'kneed' are installed",
-)
-@pytest.mark.unit
-def test_failed_imports(excel_data):
-    m = PriceTakerModel()
-    kmin = 9
-    kmax = 10
-    daily_data = m.generate_daily_data(excel_data["BaseCaseTax"])
-    with pytest.raises(
-        ImportError,
-        match=(
-            "Optimal cluster feature requires optional imports 'scikit-learn' and 'kneed'."
-        ),
-    ):
-        n_clusters, inertia_values = m.get_optimal_n_clusters(
-            daily_data, kmin=kmin, kmax=kmax
-        )
+# @pytest.mark.unit
+# def test_failed_imports(excel_data):
+#     m = PriceTakerModel()
+#     kmin = 9
+#     kmax = 10
+#     daily_data = m.generate_daily_data(excel_data["BaseCaseTax"])
+#     with pytest.raises(
+#         ImportError,
+#         match=(
+#             "Optimal cluster feature requires optional imports 'scikit-learn' and 'kneed'."
+#         ),
+#     ):
+#         n_clusters, inertia_values = m.get_optimal_n_clusters(
+#             daily_data, kmin=kmin, kmax=kmax
+#         )
 
 
 # The following test doesn't pass on all systems, so the warning for n_clusters being

setup.py

@@ -47,8 +47,6 @@ class ExtraDependencies:
     ]
     grid = [
         "gridx-prescient>=2.2.1",  # idaes.tests.prescient
-        "scikit-learn>=1.1",
-        "kneed",
     ]
     coolprop = [
         "coolprop",  # idaes.generic_models.properties.general.coolprop