cleaned up nbs and reran them all

README.md

@@ -4,10 +4,41 @@ Code and analysis used for calculating the merit order effect of renewables on p
 
 <br>
 
-### To Do:
+### Repo Publishing - To Do
 
-- [ ] Data retrieval for ENTSOE DE prices
-- [ ] Prep DE price MOE model run
-- [ ] Get the DE price MOE model running (ideally would have for tomorrow' smeeting)
-- [ ] Create mappings from fuel-type to carbon intensity for the DE and GB markets
-- [ ] Plan what I'm gonna do before Wednesday
+Notebook Polishing Changes:
+- [x] Add docstrings (can be one-liners unless shown in the user-guides or likely to be used often)
+- [x] Add a mini sentence or two at the top of each nb explaining what it's about
+- [x] Ensure there is a short explanation above each code block
+- [x] Move input data to a raw dir
+- [ ] Check all module imports are included in settings.ini
+- [x] Re-run all of the notebooks at the end to check that everything works sequentially
+
+Completed Notebooks:
+- [x] Retrieval
+- [x] EDA
+- [x] LOWESS (start with the biggy)
+- [x] Price Surface Estimation
+- [x] Price MOE
+- [x] Carbon Surface Estimation and MOE
+- [x] Prediction and Confidence Intervals
+- [x] Hyper-Parameter Tuning
+- [x] Tables and Figures
+
+New Code:
+- [ ] Separate the binder and development `environment.yml` files
+- [ ] Re-attempt LIGO fitting example as part of a user-guide
+- [ ] Add in the prediction and confidence interval plots
+- [ ] Add a lot more to the EDA examples
+- [ ] Every week re-run a single analysis (could be in the user-guide) and show the generated fit at the top of the ReadMe
+- [ ] Try to speed things up, e.g. with Numba ([one person has already started doing this](https://gist.github.com/agramfort/850437#gistcomment-3437320))
+- [ ] Get the models saved on S3 or figshare and pulled into binder via a postBuild script
+
+External/ReadMe
+- [ ] Add the GH action for version assignment triggering pypi push and zenodo update
+- [ ] Just before the paper is published set the version to 1.0.0 and have a specific Binder link that builds from that version as stored in the Zenodo archive
+- [ ] Could link the zotero collection
+- [ ] Add citations for both the external data I use and the resulting time-series I generate
+- [ ] Add bibtex citation examples for both the paper and the code (could use [this](https://citation-file-format.github.io/cff-initializer-javascript/))
+- [ ] Publish the latest version to PyPi
+- [ ] Mention the new module in the [gist](https://gist.github.com/agramfort/850437) that some of the basic regression code was inspired by 

environment.yml

@@ -26,7 +26,6 @@ dependencies:
   - pip:
       - -e .
       - ipypb
-      - feautils
       - mkdocs-material-extensions
       - mkdocstrings
       - configparser

moepy/_nbdev.py

@@ -14,12 +14,13 @@
          "parse_A44_response": "01-retrieval.ipynb",
          "retreive_DAM_prices": "01-retrieval.ipynb",
          "parse_A75_response": "01-retrieval.ipynb",
-         "retreive_production": "01-retrieval.ipynb",
+         "retrieve_production": "01-retrieval.ipynb",
          "load_EI_df": "02-eda.ipynb",
          "load_DE_df": "02-eda.ipynb",
          "clean_df_for_plot": "02-eda.ipynb",
          "rgb_2_plt_tuple": "02-eda.ipynb",
          "convert_fuel_colour_dict_to_plt_tuple": "02-eda.ipynb",
+         "hide_spines": "02-eda.ipynb",
          "stacked_fuel_plot": "02-eda.ipynb",
          "get_dist": "03-lowess.ipynb",
          "get_dist_threshold": "03-lowess.ipynb",
@@ -57,16 +58,17 @@
          "get_ensemble_preds": "03-lowess.ipynb",
          "process_smooth_dates_fit_inputs": "03-lowess.ipynb",
          "SmoothDates": "03-lowess.ipynb",
-         "PicklableFunction": "04-surface-estimation.ipynb",
-         "get_fit_kwarg_sets": "04-surface-estimation.ipynb",
-         "fit_models": "04-surface-estimation.ipynb",
+         "construct_pred_ts": "05-price-moe.ipynb",
+         "LowessDates": "03-lowess.ipynb",
+         "PicklableFunction": "04-price-surface-estimation.ipynb",
+         "get_fit_kwarg_sets": "04-price-surface-estimation.ipynb",
+         "fit_models": "04-price-surface-estimation.ipynb",
          "construct_dispatchable_lims_df": "05-price-moe.ipynb",
          "construct_pred_mask_df": "05-price-moe.ipynb",
          "AxTransformer": "05-price-moe.ipynb",
          "set_ticks": "05-price-moe.ipynb",
          "set_date_ticks": "05-price-moe.ipynb",
          "construct_df_pred": "05-price-moe.ipynb",
-         "construct_pred_ts": "05-price-moe.ipynb",
          "calc_error_metrics": "05-price-moe.ipynb",
          "get_model_pred_ts": "05-price-moe.ipynb",
          "weighted_mean_s": "05-price-moe.ipynb"}

moepy/eda.py

@@ -1,10 +1,9 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/02-eda.ipynb (unless otherwise specified).
 
 __all__ = ['load_EI_df', 'load_DE_df', 'clean_df_for_plot', 'rgb_2_plt_tuple', 'convert_fuel_colour_dict_to_plt_tuple',
-           'stacked_fuel_plot']
+           'hide_spines', 'stacked_fuel_plot']
 
 # Cell
-import json
 import pandas as pd
 
 import seaborn as sns
@@ -13,6 +12,7 @@
 
 # Cell
 def load_EI_df(EI_fp):
+    """Loads the electric insights data and returns a DataFrame"""
     df = pd.read_csv(EI_fp)
 
     df['local_datetime'] = pd.to_datetime(df['local_datetime'], utc=True)
@@ -22,6 +22,7 @@ def load_EI_df(EI_fp):
 
 # Cell
 def load_DE_df(EC_fp, ENTSOE_fp):
+    """Loads the energy-charts and ENTSOE data and returns a DataFrame"""
     # Energy-Charts
     df_DE = pd.read_csv(EC_fp)
 
@@ -44,6 +45,7 @@ def load_DE_df(EC_fp, ENTSOE_fp):
 
 # Cell
 def clean_df_for_plot(df, freq='7D'):
+    """Cleans the electric insights dataframe for plotting"""
     fuel_order = ['Imports & Storage', 'nuclear', 'biomass', 'gas', 'coal', 'hydro', 'wind', 'solar']
     interconnectors = ['french', 'irish', 'dutch', 'belgian', 'ireland', 'northern_ireland']
 
@@ -60,10 +62,12 @@ def clean_df_for_plot(df, freq='7D'):
 
 # Cell
 def rgb_2_plt_tuple(rgb_tuple):
+    """converts a standard rgb set from a 0-255 range to 0-1"""
     plt_tuple = tuple([x/255 for x in rgb_tuple])
     return plt_tuple
 
 def convert_fuel_colour_dict_to_plt_tuple(fuel_colour_dict_rgb):
+    """Converts a dictionary of fuel colours to matplotlib colour values"""
     fuel_colour_dict_plt = fuel_colour_dict_rgb.copy()
 
     fuel_colour_dict_plt = {
@@ -75,7 +79,21 @@ def convert_fuel_colour_dict_to_plt_tuple(fuel_colour_dict_rgb):
     return fuel_colour_dict_plt
 
 # Cell
+def hide_spines(ax, positions=["top", "right"]):
+    """
+    Pass a matplotlib axis and list of positions with spines to be removed
+
+    Parameters:
+        ax:          Matplotlib axis object
+        positions:   Python list e.g. ['top', 'bottom']
+    """
+    assert isinstance(positions, list), "Position must be passed as a list "
+
+    for position in positions:
+        ax.spines[position].set_visible(False)
+
 def stacked_fuel_plot(df, fuel_colour_dict, ax=None, save_path=None, dpi=150):
+    """Plots the electric insights fuel data as a stacked area graph"""
     df = df[fuel_colour_dict.keys()]
 
     if ax == None:
@@ -86,8 +104,7 @@ def stacked_fuel_plot(df, fuel_colour_dict, ax=None, save_path=None, dpi=150):
 
     plt.rcParams['axes.ymargin'] = 0
     ax.spines['bottom'].set_position('zero')
-    ax.spines['right'].set_visible(False)
-    ax.spines['top'].set_visible(False)
+    hide_spines(ax)
 
     ax.set_xlim(df.index.min(), df.index.max())
     ax.legend(ncol=4, bbox_to_anchor=(0.85, 1.15), frameon=False)