working on new skopt integration

environment.yml

@@ -20,6 +20,7 @@ dependencies:
   - dagit
   - statsmodels
   - scikit-learn
+  - scikit-optimize
   - pip
 
   - pip:

moepy/_nbdev.py

@@ -55,6 +55,7 @@
          "construct_dt_weights": "03-lowess.ipynb",
          "fit_external_weighted_ensemble": "03-lowess.ipynb",
          "get_ensemble_preds": "03-lowess.ipynb",
+         "process_smooth_dates_fit_inputs": "03-lowess.ipynb",
          "SmoothDates": "03-lowess.ipynb",
          "PicklableFunction": "04-surface-estimation.ipynb",
          "get_fit_kwarg_sets": "04-surface-estimation.ipynb",

moepy/lowess.py

@@ -8,7 +8,7 @@
            'get_bootstrap_resid_std_devs', 'run_model', 'bootstrap_model', 'get_confidence_interval',
            'pred_to_quantile_loss', 'calc_quant_reg_loss', 'calc_quant_reg_betas', 'quantile_model',
            'calc_timedelta_dists', 'construct_dt_weights', 'fit_external_weighted_ensemble', 'get_ensemble_preds',
-           'SmoothDates']
+           'process_smooth_dates_fit_inputs', 'SmoothDates']
 
 # Cell
 import pandas as pd
@@ -434,17 +434,36 @@ def get_ensemble_preds(ensemble_member_to_model, x_pred=np.linspace(8, 60, 53)):
     return ensemble_member_to_preds
 
 # Cell
+def process_smooth_dates_fit_inputs(x, y, dt_idx, reg_dates):
+    if hasattr(x, 'index') and hasattr(y, 'index'):
+        assert x.index.equals(y.index), 'If `x` and `y` have indexes then they must be the same'
+        if dt_idx is None:
+            dt_idx = x.index
+
+        x = x.values
+        y = y.values
+
+    assert dt_idx is not None, '`dt_idx` must either be passed directly or `x` and `y` must include indexes'
+
+    if reg_dates is None:
+        reg_dates = dt_idx
+
+    return x, y, dt_idx, reg_dates
+
 class SmoothDates(BaseEstimator, RegressorMixin):
-    def __init__(self):
+    def __init__(self, frac=0.3, threshold_value=52, threshold_units='W'):
         self.fitted = False
-        pass
+        self.frac = frac
+        self.threshold_value = threshold_value
+        self.threshold_units = threshold_units
 
-    def fit(self, x, y, dt_idx, reg_dates, threshold_value=52, threshold_units='W', lowess_kwargs={}, **fit_kwargs):
+    def fit(self, x, y, dt_idx=None, reg_dates=None, lowess_kwargs={}, **fit_kwargs):
+        x, y, dt_idx, reg_dates = process_smooth_dates_fit_inputs(x, y, dt_idx, reg_dates)
         self.ensemble_member_to_weights = construct_dt_weights(dt_idx, reg_dates,
-                                                               threshold_value=threshold_value,
-                                                               threshold_units=threshold_units)
+                                                               threshold_value=self.threshold_value,
+                                                               threshold_units=self.threshold_units)
 
-        self.ensemble_member_to_models = fit_external_weighted_ensemble(x, y, self.ensemble_member_to_weights, lowess_kwargs=lowess_kwargs, **fit_kwargs)
+        self.ensemble_member_to_models = fit_external_weighted_ensemble(x, y, self.ensemble_member_to_weights, lowess_kwargs=lowess_kwargs, frac=self.frac, **fit_kwargs)
 
         self.reg_dates = reg_dates
         self.fitted = True
@@ -455,6 +474,9 @@ def predict(self, x_pred=np.linspace(8, 60, 53), dt_pred=None, return_df=True):
         if dt_pred is None:
             dt_pred = self.reg_dates
 
+        if isinstance(x_pred, pd.Series):
+            x_pred = x_pred.values
+
         self.ensemble_member_to_preds = get_ensemble_preds(self.ensemble_member_to_models, x_pred=x_pred)
 
         self.pred_weights = np.array(list(construct_dt_weights(dt_pred, self.reg_dates).values()))

moepy/moe.py

@@ -9,6 +9,7 @@
 import numpy as np
 
 import pickle
+import scipy
 from sklearn import linear_model
 from sklearn.metrics import r2_score
 from collections.abc import Iterable

nbs/03-lowess.ipynb

@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -2531,7 +2531,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 71,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -5083,22 +5083,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 353,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
     "#exports\n",
+    "def process_smooth_dates_fit_inputs(x, y, dt_idx, reg_dates):  \n",
+    "    if hasattr(x, 'index') and hasattr(y, 'index'):\n",
+    "        assert x.index.equals(y.index), 'If `x` and `y` have indexes then they must be the same'\n",
+    "        if dt_idx is None:\n",
+    "            dt_idx = x.index\n",
+    "\n",
+    "        x = x.values\n",
+    "        y = y.values\n",
+    "\n",
+    "    assert dt_idx is not None, '`dt_idx` must either be passed directly or `x` and `y` must include indexes'\n",
+    "\n",
+    "    if reg_dates is None:\n",
+    "        reg_dates = dt_idx\n",
+    "    \n",
+    "    return x, y, dt_idx, reg_dates\n",
+    "\n",
     "class SmoothDates(BaseEstimator, RegressorMixin):\n",
-    "    def __init__(self):\n",
+    "    def __init__(self, frac=0.3, threshold_value=52, threshold_units='W'):\n",
     "        self.fitted = False\n",
-    "        pass\n",
+    "        self.frac = frac\n",
+    "        self.threshold_value = threshold_value\n",
+    "        self.threshold_units = threshold_units\n",
     "    \n",
-    "    def fit(self, x, y, dt_idx, reg_dates, threshold_value=52, threshold_units='W', lowess_kwargs={}, **fit_kwargs):        \n",
+    "    def fit(self, x, y, dt_idx=None, reg_dates=None, lowess_kwargs={}, **fit_kwargs):          \n",
+    "        x, y, dt_idx, reg_dates = process_smooth_dates_fit_inputs(x, y, dt_idx, reg_dates)\n",
     "        self.ensemble_member_to_weights = construct_dt_weights(dt_idx, reg_dates, \n",
-    "                                                               threshold_value=threshold_value, \n",
-    "                                                               threshold_units=threshold_units)\n",
+    "                                                               threshold_value=self.threshold_value, \n",
+    "                                                               threshold_units=self.threshold_units)\n",
     "        \n",
-    "        self.ensemble_member_to_models = fit_external_weighted_ensemble(x, y, self.ensemble_member_to_weights, lowess_kwargs=lowess_kwargs, **fit_kwargs)\n",
+    "        self.ensemble_member_to_models = fit_external_weighted_ensemble(x, y, self.ensemble_member_to_weights, lowess_kwargs=lowess_kwargs, frac=self.frac, **fit_kwargs)\n",
     "        \n",
     "        self.reg_dates = reg_dates\n",
     "        self.fitted = True\n",
@@ -5109,6 +5128,9 @@
     "        if dt_pred is None:\n",
     "            dt_pred = self.reg_dates\n",
     "            \n",
+    "        if isinstance(x_pred, pd.Series):\n",
+    "            x_pred = x_pred.values\n",
+    "            \n",
     "        self.ensemble_member_to_preds = get_ensemble_preds(self.ensemble_member_to_models, x_pred=x_pred)\n",
     "        \n",
     "        self.pred_weights = np.array(list(construct_dt_weights(dt_pred, self.reg_dates).values()))\n",
@@ -5417,7 +5439,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 400,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -5427,7 +5449,12 @@
       "Converted 01-retrieval.ipynb.\n",
       "Converted 02-eda.ipynb.\n",
       "Converted 03-lowess.ipynb.\n",
-      "Converted 04-merit-order-effect.ipynb.\n"
+      "Converted 04-surface-estimation.ipynb.\n",
+      "Converted 05-price-moe.ipynb.\n",
+      "Converted 06-carbon-moe.ipynb.\n",
+      "Converted 07-pred-conf-intvls.ipynb.\n",
+      "Converted 08-hyper-param-tuning.ipynb.\n",
+      "Converted 09-tables.ipynb.\n"
      ]
     }
    ],