diff --git a/README.md b/README.md
index 9d13168..4a8ea34 100644
--- a/README.md
+++ b/README.md
@@ -4,10 +4,41 @@ Code and analysis used for calculating the merit order effect of renewables on p
-### To Do:
+### Repo Publishing - To Do
-- [ ] Data retrieval for ENTSOE DE prices
-- [ ] Prep DE price MOE model run
-- [ ] Get the DE price MOE model running (ideally would have for tomorrow' smeeting)
-- [ ] Create mappings from fuel-type to carbon intensity for the DE and GB markets
-- [ ] Plan what I'm gonna do before Wednesday
\ No newline at end of file
+Notebook Polishing Changes:
+- [x] Add docstrings (can be one-liners unless shown in the user-guides or likely to be used often)
+- [x] Add a mini sentence or two at the top of each nb explaining what it's about
+- [x] Ensure there is a short explanation above each code block
+- [x] Move input data to a raw dir
+- [ ] Check all module imports are included in settings.ini
+- [x] Re-run all of the notebooks at the end to check that everything works sequentially
+
+Completed Notebooks:
+- [x] Retrieval
+- [x] EDA
+- [x] LOWESS (start with the biggy)
+- [x] Price Surface Estimation
+- [x] Price MOE
+- [x] Carbon Surface Estimation and MOE
+- [x] Prediction and Confidence Intervals
+- [x] Hyper-Parameter Tuning
+- [x] Tables and Figures
+
+New Code:
+- [ ] Separate the binder and development `environment.yml` files
+- [ ] Re-attempt LIGO fitting example as part of a user-guide
+- [ ] Add in the prediction and confidence interval plots
+- [ ] Add a lot more to the EDA examples
+- [ ] Every week re-run a single analysis (could be in the user-guide) and show the generated fit at the top of the ReadMe
+- [ ] Try to speed things up, e.g. with Numba ([one person has already started doing this](https://gist.github.com/agramfort/850437#gistcomment-3437320))
+- [ ] Get the models saved on S3 or figshare and pulled into binder via a postBuild script
+
+External/ReadMe
+- [ ] Add the GH action for version assignment triggering pypi push and zenodo update
+- [ ] Just before the paper is published set the version to 1.0.0 and have a specific Binder link that builds from that version as stored in the Zenodo archive
+- [ ] Could link the zotero collection
+- [ ] Add citations for both the external data I use and the resulting time-series I generate
+- [ ] Add bibtex citation examples for both the paper and the code (could use [this](https://citation-file-format.github.io/cff-initializer-javascript/))
+- [ ] Publish the latest version to PyPi
+- [ ] Mention the new module in the [gist](https://gist.github.com/agramfort/850437) that some of the basic regression code was inspired by
\ No newline at end of file
diff --git a/data/EI_rename_mapping.json b/data/EI_rename_mapping.json
deleted file mode 100644
index 1d1890f..0000000
--- a/data/EI_rename_mapping.json
+++ /dev/null
@@ -1 +0,0 @@
-{"pumpedStorage": "pumped_storage", "northernIreland": "northern_ireland", "windOnshore": "wind_onshore", "windOffshore": "wind_offshore", "prices_ahead": "day_ahead_price", "prices": "imbalance_price", "temperatures": "temperature", "totalInGperkWh": "gCO2_per_kWh", "totalInTperh": "TCO2_per_h"}
\ No newline at end of file
diff --git a/data/fuel_colours.json b/data/fuel_colours.json
deleted file mode 100644
index fd547ca..0000000
--- a/data/fuel_colours.json
+++ /dev/null
@@ -1 +0,0 @@
-{"Imports & Storage": [121, 68, 149], "nuclear": [77, 157, 87], "biomass": [168, 125, 81], "gas": [254, 156, 66], "coal": [122, 122, 122], "hydro": [50, 120, 196], "wind": [72, 194, 227], "solar": [255, 219, 65]}
\ No newline at end of file
diff --git a/data/ENTSOE_DE_price.csv b/data/raw/ENTSOE_DE_price.csv
similarity index 100%
rename from data/ENTSOE_DE_price.csv
rename to data/raw/ENTSOE_DE_price.csv
diff --git a/data/electric_insights.csv b/data/raw/electric_insights.csv
similarity index 100%
rename from data/electric_insights.csv
rename to data/raw/electric_insights.csv
diff --git a/data/energy_charts.csv b/data/raw/energy_charts.csv
similarity index 100%
rename from data/energy_charts.csv
rename to data/raw/energy_charts.csv
diff --git a/environment.yml b/environment.yml
index 29d8abf..483bec2 100644
--- a/environment.yml
+++ b/environment.yml
@@ -26,7 +26,6 @@ dependencies:
- pip:
- -e .
- ipypb
- - feautils
- mkdocs-material-extensions
- mkdocstrings
- configparser
diff --git a/img/2D_skopt_surface.png b/img/2D_skopt_surface.png
new file mode 100644
index 0000000..9ec9d72
Binary files /dev/null and b/img/2D_skopt_surface.png differ
diff --git a/img/2D_skopt_surface_DE.png b/img/2D_skopt_surface_DE.png
new file mode 100644
index 0000000..c2b1a52
Binary files /dev/null and b/img/2D_skopt_surface_DE.png differ
diff --git a/img/LOWESS_single_regression_example.png b/img/LOWESS_single_regression_example.png
new file mode 100644
index 0000000..26ef149
Binary files /dev/null and b/img/LOWESS_single_regression_example.png differ
diff --git a/img/tricube_weighting_diagram.png b/img/tricube_weighting_diagram.png
new file mode 100644
index 0000000..1341e78
Binary files /dev/null and b/img/tricube_weighting_diagram.png differ
diff --git a/moepy/_nbdev.py b/moepy/_nbdev.py
index 1faf2b1..10bf655 100644
--- a/moepy/_nbdev.py
+++ b/moepy/_nbdev.py
@@ -14,12 +14,13 @@
"parse_A44_response": "01-retrieval.ipynb",
"retreive_DAM_prices": "01-retrieval.ipynb",
"parse_A75_response": "01-retrieval.ipynb",
- "retreive_production": "01-retrieval.ipynb",
+ "retrieve_production": "01-retrieval.ipynb",
"load_EI_df": "02-eda.ipynb",
"load_DE_df": "02-eda.ipynb",
"clean_df_for_plot": "02-eda.ipynb",
"rgb_2_plt_tuple": "02-eda.ipynb",
"convert_fuel_colour_dict_to_plt_tuple": "02-eda.ipynb",
+ "hide_spines": "02-eda.ipynb",
"stacked_fuel_plot": "02-eda.ipynb",
"get_dist": "03-lowess.ipynb",
"get_dist_threshold": "03-lowess.ipynb",
@@ -57,16 +58,17 @@
"get_ensemble_preds": "03-lowess.ipynb",
"process_smooth_dates_fit_inputs": "03-lowess.ipynb",
"SmoothDates": "03-lowess.ipynb",
- "PicklableFunction": "04-surface-estimation.ipynb",
- "get_fit_kwarg_sets": "04-surface-estimation.ipynb",
- "fit_models": "04-surface-estimation.ipynb",
+ "construct_pred_ts": "05-price-moe.ipynb",
+ "LowessDates": "03-lowess.ipynb",
+ "PicklableFunction": "04-price-surface-estimation.ipynb",
+ "get_fit_kwarg_sets": "04-price-surface-estimation.ipynb",
+ "fit_models": "04-price-surface-estimation.ipynb",
"construct_dispatchable_lims_df": "05-price-moe.ipynb",
"construct_pred_mask_df": "05-price-moe.ipynb",
"AxTransformer": "05-price-moe.ipynb",
"set_ticks": "05-price-moe.ipynb",
"set_date_ticks": "05-price-moe.ipynb",
"construct_df_pred": "05-price-moe.ipynb",
- "construct_pred_ts": "05-price-moe.ipynb",
"calc_error_metrics": "05-price-moe.ipynb",
"get_model_pred_ts": "05-price-moe.ipynb",
"weighted_mean_s": "05-price-moe.ipynb"}
diff --git a/moepy/eda.py b/moepy/eda.py
index f97b6ba..e0647b5 100644
--- a/moepy/eda.py
+++ b/moepy/eda.py
@@ -1,10 +1,9 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/02-eda.ipynb (unless otherwise specified).
__all__ = ['load_EI_df', 'load_DE_df', 'clean_df_for_plot', 'rgb_2_plt_tuple', 'convert_fuel_colour_dict_to_plt_tuple',
- 'stacked_fuel_plot']
+ 'hide_spines', 'stacked_fuel_plot']
# Cell
-import json
import pandas as pd
import seaborn as sns
@@ -13,6 +12,7 @@
# Cell
def load_EI_df(EI_fp):
+ """Loads the electric insights data and returns a DataFrame"""
df = pd.read_csv(EI_fp)
df['local_datetime'] = pd.to_datetime(df['local_datetime'], utc=True)
@@ -22,6 +22,7 @@ def load_EI_df(EI_fp):
# Cell
def load_DE_df(EC_fp, ENTSOE_fp):
+ """Loads the energy-charts and ENTSOE data and returns a DataFrame"""
# Energy-Charts
df_DE = pd.read_csv(EC_fp)
@@ -44,6 +45,7 @@ def load_DE_df(EC_fp, ENTSOE_fp):
# Cell
def clean_df_for_plot(df, freq='7D'):
+ """Cleans the electric insights dataframe for plotting"""
fuel_order = ['Imports & Storage', 'nuclear', 'biomass', 'gas', 'coal', 'hydro', 'wind', 'solar']
interconnectors = ['french', 'irish', 'dutch', 'belgian', 'ireland', 'northern_ireland']
@@ -60,10 +62,12 @@ def clean_df_for_plot(df, freq='7D'):
# Cell
def rgb_2_plt_tuple(rgb_tuple):
+ """converts a standard rgb set from a 0-255 range to 0-1"""
plt_tuple = tuple([x/255 for x in rgb_tuple])
return plt_tuple
def convert_fuel_colour_dict_to_plt_tuple(fuel_colour_dict_rgb):
+ """Converts a dictionary of fuel colours to matplotlib colour values"""
fuel_colour_dict_plt = fuel_colour_dict_rgb.copy()
fuel_colour_dict_plt = {
@@ -75,7 +79,21 @@ def convert_fuel_colour_dict_to_plt_tuple(fuel_colour_dict_rgb):
return fuel_colour_dict_plt
# Cell
+def hide_spines(ax, positions=["top", "right"]):
+ """
+ Pass a matplotlib axis and list of positions with spines to be removed
+
+ Parameters:
+ ax: Matplotlib axis object
+ positions: Python list e.g. ['top', 'bottom']
+ """
+ assert isinstance(positions, list), "Position must be passed as a list "
+
+ for position in positions:
+ ax.spines[position].set_visible(False)
+
def stacked_fuel_plot(df, fuel_colour_dict, ax=None, save_path=None, dpi=150):
+ """Plots the electric insights fuel data as a stacked area graph"""
df = df[fuel_colour_dict.keys()]
if ax == None:
@@ -86,8 +104,7 @@ def stacked_fuel_plot(df, fuel_colour_dict, ax=None, save_path=None, dpi=150):
plt.rcParams['axes.ymargin'] = 0
ax.spines['bottom'].set_position('zero')
- ax.spines['right'].set_visible(False)
- ax.spines['top'].set_visible(False)
+ hide_spines(ax)
ax.set_xlim(df.index.min(), df.index.max())
ax.legend(ncol=4, bbox_to_anchor=(0.85, 1.15), frameon=False)
diff --git a/moepy/lowess.py b/moepy/lowess.py
index 5bf7d74..881ebca 100644
--- a/moepy/lowess.py
+++ b/moepy/lowess.py
@@ -8,7 +8,7 @@
'get_bootstrap_resid_std_devs', 'run_model', 'bootstrap_model', 'get_confidence_interval',
'pred_to_quantile_loss', 'calc_quant_reg_loss', 'calc_quant_reg_betas', 'quantile_model',
'calc_timedelta_dists', 'construct_dt_weights', 'fit_external_weighted_ensemble', 'get_ensemble_preds',
- 'process_smooth_dates_fit_inputs', 'SmoothDates']
+ 'process_smooth_dates_fit_inputs', 'SmoothDates', 'construct_pred_ts', 'LowessDates']
# Cell
import pandas as pd
@@ -24,7 +24,6 @@
from scipy import linalg
from timeit import timeit
-import FEAutils as hlp
from ipypb import track
from moepy import eda
@@ -34,6 +33,7 @@
# Cell
def get_dist_threshold(dist, frac=0.4):
+ """Identifies the minimum distance that contains the desired data fraction"""
frac_idx = int(np.ceil(len(dist)*frac))
dist_threshold = sorted(dist)[frac_idx]
@@ -44,6 +44,7 @@ def get_dist_threshold(dist, frac=0.4):
# Cell
def get_all_weights(x, frac=0.4):
+ """Calculates the weightings at each data point for a LOWESS regression"""
all_weights = []
for i in range(len(x)):
@@ -65,12 +66,16 @@ def get_all_weights(x, frac=0.4):
# Cell
def clean_weights(weights):
- weights = weights/weights.sum(axis=0) # We'll then normalise the weights so that for each model they sum to 1 for a single data point
+ """Normalises each models weightings and removes non-finite values"""
+ with np.errstate(divide='ignore', invalid='ignore'):
+ weights = weights/weights.sum(axis=0) # We'll then normalise the weights so that for each model they sum to 1 for a single data point
+
weights = np.where(~np.isfinite(weights), 0, weights) # And remove any non-finite values
return weights
def dist_2_weights_matrix(dist_matrix, dist_thresholds):
+ """Converts distance matrix and thresholds to weightings"""
weights = dist_to_weights(dist_matrix, dist_thresholds.reshape(-1, 1))
weights = clean_weights(weights)
@@ -78,6 +83,7 @@ def dist_2_weights_matrix(dist_matrix, dist_thresholds):
# Cell
def get_full_dataset_weights_matrix(x, frac=0.4):
+ """Wrapper for calculating weights from the raw data and LOWESS fraction"""
frac_idx = get_frac_idx(x, frac)
dist_matrix = vector_to_dist_matrix(x)
@@ -91,6 +97,7 @@ def get_full_dataset_weights_matrix(x, frac=0.4):
num_fits_2_reg_anchors = lambda x, num_fits: np.linspace(x.min(), x.max(), num=num_fits)
def get_weighting_locs(x, reg_anchors=None, num_fits=None):
+ """Identifies the weighting locations for the provided dataset"""
num_type_2_dist_rows = {
type(None) : lambda x, num_fits: x.reshape(-1, 1),
int : lambda x, num_fits: num_fits_2_reg_anchors(x, num_fits).reshape(-1, 1),
@@ -104,6 +111,7 @@ def get_weighting_locs(x, reg_anchors=None, num_fits=None):
return weighting_locs
def create_dist_matrix(x, reg_anchors=None, num_fits=None):
+ """Constructs the distance matrix for the desired weighting locations"""
weighting_locs = get_weighting_locs(x, reg_anchors=reg_anchors, num_fits=num_fits)
dist_matrix = np.abs(weighting_locs - x.reshape(1, -1))
@@ -111,6 +119,7 @@ def create_dist_matrix(x, reg_anchors=None, num_fits=None):
# Cell
def get_weights_matrix(x, frac=0.4, weighting_locs=None, reg_anchors=None, num_fits=None):
+ """Wrapper for calculating weights from the raw data and LOWESS fraction"""
frac_idx = get_frac_idx(x, frac)
if weighting_locs is not None:
@@ -125,6 +134,7 @@ def get_weights_matrix(x, frac=0.4, weighting_locs=None, reg_anchors=None, num_f
# Cell
def calc_lin_reg_betas(x, y, weights=None):
+ """Calculates the intercept and gradient for the specified local regressions"""
if weights is None:
weights = np.ones(len(x))
@@ -132,7 +142,7 @@ def calc_lin_reg_betas(x, y, weights=None):
A = np.array([[np.sum(weights), np.sum(weights * x)],
[np.sum(weights * x), np.sum(weights * x * x)]])
- betas = linalg.solve(A, b)
+ betas = np.linalg.lstsq(A, b)[0]
return betas
@@ -140,6 +150,7 @@ def calc_lin_reg_betas(x, y, weights=None):
check_array = lambda array, x: np.ones(len(x)) if array is None else array
def fit_regressions(x, y, weights=None, reg_func=calc_lin_reg_betas, num_coef=2, **reg_params):
+ """Calculates the design matrix for the specified local regressions"""
if weights is None:
weights = np.ones(len(x))
@@ -155,6 +166,7 @@ def fit_regressions(x, y, weights=None, reg_func=calc_lin_reg_betas, num_coef=2,
# Cell
def lowess_fit_and_predict(x, y, frac=0.4, reg_anchors=None, num_fits=None, x_pred=None):
+ """Fits and predicts smoothed local regressions at the specified locations"""
weighting_locs = get_weighting_locs(x, reg_anchors=reg_anchors, num_fits=num_fits)
weights = get_weights_matrix(x, frac=frac, weighting_locs=weighting_locs)
design_matrix = fit_regressions(x, y, weights)
@@ -171,6 +183,7 @@ def lowess_fit_and_predict(x, y, frac=0.4, reg_anchors=None, num_fits=None, x_pr
# Cell
def calc_robust_weights(y, y_pred, max_std_dev=6):
+ """Calculates robustifying weightings that penalise outliers"""
residuals = y - y_pred
std_dev = np.quantile(np.abs(residuals), 0.682)
@@ -181,6 +194,7 @@ def calc_robust_weights(y, y_pred, max_std_dev=6):
# Cell
def robust_lowess_fit_and_predict(x, y, frac=0.4, reg_anchors=None, num_fits=None, x_pred=None, robust_weights=None, robust_iters=3):
+ """Fits and predicts robust smoothed local regressions at the specified locations"""
# Identifying the initial loading weights
weighting_locs = get_weighting_locs(x, reg_anchors=reg_anchors, num_fits=num_fits)
loading_weights = get_weights_matrix(x, frac=frac, weighting_locs=weighting_locs)
@@ -190,7 +204,10 @@ def robust_lowess_fit_and_predict(x, y, frac=0.4, reg_anchors=None, num_fits=Non
robust_loading_weights = loading_weights
else:
robust_loading_weights = np.multiply(robust_weights, loading_weights)
- robust_loading_weights = robust_loading_weights/robust_loading_weights.sum(axis=0)
+
+ with np.errstate(divide='ignore', invalid='ignore'):
+ robust_loading_weights = robust_loading_weights/robust_loading_weights.sum(axis=0)
+
robust_loading_weights = np.where(~np.isfinite(robust_loading_weights), 0, robust_loading_weights)
# Fitting the model and making predictions
@@ -215,12 +232,61 @@ def robust_lowess_fit_and_predict(x, y, frac=0.4, reg_anchors=None, num_fits=Non
# Cell
class Lowess(BaseEstimator, RegressorMixin):
+ """
+ This class provides a Scikit-Learn compatible model for Locally Weighted
+ Scatterplot Smoothing, including robustifying procedures against outliers.
+
+ For more information on the underlying algorithm please refer to
+ * William S. Cleveland: "Robust locally weighted regression and smoothing
+ scatterplots", Journal of the American Statistical Association, December 1979,
+ volume 74, number 368, pp. 829-836.
+ * William S. Cleveland and Susan J. Devlin: "Locally weighted regression: An
+ approach to regression analysis by local fitting", Journal of the American
+ Statistical Association, September 1988, volume 83, number 403, pp. 596-610.
+
+ Example Usage:
+ ```
+ x = np.linspace(0, 5, num=150)
+ y = np.sin(x)
+ y_noisy = y + (np.random.normal(size=len(y)))/10
+
+ lowess = Lowess()
+ lowess.fit(x, y_noisy, frac=0.2)
+
+ x_pred = np.linspace(0, 5, 26)
+ y_pred = lowess.predict(x_pred)
+ ```
+
+ Initialisation Parameters:
+ reg_func: function that accepts the x and y values then returns the intercepts and gradients
+
+ Attributes:
+ reg_func: function that accepts the x and y values then returns the intercepts and gradients
+ fitted: Boolean flag indicating whether the model has been fitted
+ frac: Fraction of the dataset to use in each local regression
+ weighting_locs: Locations of the local regression centers
+ loading_weights: Weights of each data-point across the localalised models
+ design_matrix: Regression coefficients for each of the localised models
+ """
+
def __init__(self, reg_func=calc_lin_reg_betas):
self.reg_func = reg_func
self.fitted = False
return
+
def calculate_loading_weights(self, x, reg_anchors=None, num_fits=None, external_weights=None, robust_weights=None):
+ """
+ Calculates the loading weights for each data-point across the localised models
+
+ Parameters:
+ x: values for the independent variable
+ reg_anchors: Locations at which to center the local regressions
+ num_fits: Number of locations at which to carry out a local regression
+ external_weights: Further weighting for the specific regression
+ robust_weights: Robustifying weights to remove the influence of outliers
+ """
+
# Calculating the initial loading weights
weighting_locs = get_weighting_locs(x, reg_anchors=reg_anchors, num_fits=num_fits)
loading_weights = get_weights_matrix(x, frac=self.frac, weighting_locs=weighting_locs)
@@ -236,7 +302,9 @@ def calculate_loading_weights(self, x, reg_anchors=None, num_fits=None, external
loading_weights = np.multiply(weight_adj, loading_weights)
# Post-processing weights
- loading_weights = loading_weights/loading_weights.sum(axis=0) # normalising
+ with np.errstate(divide='ignore', invalid='ignore'):
+ loading_weights = loading_weights/loading_weights.sum(axis=0) # normalising
+
loading_weights = np.where(~np.isfinite(loading_weights), 0, loading_weights) # removing non-finite values
self.weighting_locs = weighting_locs
@@ -244,7 +312,27 @@ def calculate_loading_weights(self, x, reg_anchors=None, num_fits=None, external
return
- def fit(self, x, y, frac=0.4, reg_anchors=None, num_fits=None, external_weights=None, robust_weights=None, robust_iters=3, **reg_params):
+
+ def fit(self, x, y, frac=0.4, reg_anchors=None,
+ num_fits=None, external_weights=None,
+ robust_weights=None, robust_iters=3, **reg_params):
+ """
+ Calculation of the local regression coefficients for
+ a LOWESS model across the dataset provided. This method
+ will reassign the `frac`, `weighting_locs`, `loading_weights`,
+ and `design_matrix` attributes of the `Lowess` object.
+
+ Parameters:
+ x: values for the independent variable
+ y: values for the dependent variable
+ frac: LOWESS bandwidth for local regression as a fraction
+ reg_anchors: Locations at which to center the local regressions
+ num_fits: Number of locations at which to carry out a local regression
+ external_weights: Further weighting for the specific regression
+ robust_weights: Robustifying weights to remove the influence of outliers
+ robust_iters: Number of robustifying iterations to carry out
+ """
+
self.frac = frac
# Solving for the design matrix
@@ -265,7 +353,18 @@ def fit(self, x, y, frac=0.4, reg_anchors=None, num_fits=None, external_weights=
return
+
def predict(self, x_pred):
+ """
+ Inference using the design matrix from the LOWESS fit
+
+ Parameters:
+ x_pred: Locations for the LOWESS inference
+
+ Returns:
+ y_pred: Estimated values using the LOWESS fit
+ """
+
point_evals = self.design_matrix[:, 0] + np.dot(x_pred.reshape(-1, 1), self.design_matrix[:, 1].reshape(1, -1))
pred_weights = get_weights_matrix(x_pred, frac=self.frac, reg_anchors=self.weighting_locs)
@@ -275,7 +374,8 @@ def predict(self, x_pred):
# Cell
def get_bootstrap_idxs(x, bootstrap_bag_size=0.5):
- ## Bag size handling
+ """Determines the indexes of an array to be used for the in- and out-of-bag bootstrap samples"""
+ # Bag size handling
assert bootstrap_bag_size>0, 'Bootstrap bag size must be greater than 0'
if bootstrap_bag_size > 1:
@@ -284,7 +384,7 @@ def get_bootstrap_idxs(x, bootstrap_bag_size=0.5):
else:
bootstrap_bag_size = int(np.ceil(bootstrap_bag_size*len(x)))
- ## Splitting in-bag and out-of-bag samlpes
+ # Splitting in-bag and out-of-bag samlpes
idxs = np.array(range(len(x)))
ib_idxs = np.sort(np.random.choice(idxs, bootstrap_bag_size, replace=True))
@@ -294,6 +394,7 @@ def get_bootstrap_idxs(x, bootstrap_bag_size=0.5):
# Cell
def get_bootstrap_resid_std_devs(x, y, bag_size, model=Lowess(), **model_kwargs):
+ """Calculates the standard deviation of the in- and out-of-bag errors"""
# Splitting the in- and out-of-bag samples
ib_idxs, oob_idxs = get_bootstrap_idxs(x, bag_size)
@@ -318,6 +419,7 @@ def get_bootstrap_resid_std_devs(x, y, bag_size, model=Lowess(), **model_kwargs)
# Cell
def run_model(x, y, bag_size, model=Lowess(), x_pred=None, **model_kwargs):
+ """Fits a model and then uses it to make a prediction"""
if x_pred is None:
x_pred = x
@@ -332,6 +434,7 @@ def run_model(x, y, bag_size, model=Lowess(), x_pred=None, **model_kwargs):
return y_pred
def bootstrap_model(x, y, bag_size=0.5, model=Lowess(), x_pred=None, num_runs=1000, **model_kwargs):
+ """Repeatedly fits and predicts using the specified model, using different subsets of the data each time"""
# Creating the ensemble predictions
preds = []
@@ -349,6 +452,7 @@ def bootstrap_model(x, y, bag_size=0.5, model=Lowess(), x_pred=None, num_runs=10
# Cell
def get_confidence_interval(df_bootstrap, conf_pct=0.95):
+ """Estimates the confidence interval of a prediction based on the bootstrapped estimates"""
conf_margin = (1 - conf_pct)/2
df_conf_intvl = pd.DataFrame(columns=['min', 'max'], index=df_bootstrap.index)
@@ -359,6 +463,7 @@ def get_confidence_interval(df_bootstrap, conf_pct=0.95):
# Cell
def pred_to_quantile_loss(y, y_pred, q=0.5, weights=None):
+ """Calculates the quantile error for a prediction"""
residuals = y - y_pred
if weights is not None:
@@ -369,6 +474,7 @@ def pred_to_quantile_loss(y, y_pred, q=0.5, weights=None):
return loss
def calc_quant_reg_loss(x0, x, y, q, weights=None):
+ """Makes a quantile prediction then calculates its error"""
if weights is None:
weights = np.ones(len(x))
@@ -382,6 +488,7 @@ def calc_quant_reg_loss(x0, x, y, q, weights=None):
# Cell
def quantile_model(x, y, model=Lowess(calc_quant_reg_betas),
x_pred=None, qs=np.linspace(0.1, 0.9, 9), **model_kwargs):
+ """Model wrapper that will repeatedly fit and predict for the specified quantiles"""
if x_pred is None:
x_pred = np.sort(np.unique(x))
@@ -401,6 +508,7 @@ def quantile_model(x, y, model=Lowess(calc_quant_reg_betas),
# Cell
def calc_timedelta_dists(dates, central_date, threshold_value=24, threshold_units='W'):
+ """Maps datetimes to weights using the central date and threshold information provided"""
timedeltas = pd.to_datetime(dates, utc=True) - pd.to_datetime(central_date, utc=True)
timedelta_dists = timedeltas/pd.Timedelta(value=threshold_value, unit=threshold_units)
@@ -408,6 +516,7 @@ def calc_timedelta_dists(dates, central_date, threshold_value=24, threshold_unit
# Cell
def construct_dt_weights(dt_idx, reg_dates, threshold_value=52, threshold_units='W'):
+ """Constructs a set of distance weightings based on the regression dates provided"""
dt_to_weights = dict()
for reg_date in reg_dates:
@@ -417,6 +526,7 @@ def construct_dt_weights(dt_idx, reg_dates, threshold_value=52, threshold_units=
# Cell
def fit_external_weighted_ensemble(x, y, ensemble_member_to_weights, lowess_kwargs={}, **fit_kwargs):
+ """Fits an ensemble of LOWESS models which have varying relevance for each subset of data over time"""
ensemble_member_to_models = dict()
for ensemble_member, ensemble_weights in track(ensemble_member_to_weights.items()):
@@ -426,6 +536,7 @@ def fit_external_weighted_ensemble(x, y, ensemble_member_to_weights, lowess_kwar
return ensemble_member_to_models
def get_ensemble_preds(ensemble_member_to_model, x_pred=np.linspace(8, 60, 53)):
+ """Using the fitted ensemble of LOWESS models to generate the predictions for each of them"""
ensemble_member_to_preds = dict()
for ensemble_member in ensemble_member_to_model.keys():
@@ -433,8 +544,8 @@ def get_ensemble_preds(ensemble_member_to_model, x_pred=np.linspace(8, 60, 53)):
return ensemble_member_to_preds
-# Cell
def process_smooth_dates_fit_inputs(x, y, dt_idx, reg_dates):
+ """Sanitises the inputs to the SmoothDates fitting method"""
if hasattr(x, 'index') and hasattr(y, 'index'):
assert x.index.equals(y.index), 'If `x` and `y` have indexes then they must be the same'
if dt_idx is None:
@@ -450,14 +561,57 @@ def process_smooth_dates_fit_inputs(x, y, dt_idx, reg_dates):
return x, y, dt_idx, reg_dates
+# Cell
class SmoothDates(BaseEstimator, RegressorMixin):
+ """
+ This class provides a time-adaptive extension of the classical
+ Locally Weighted Scatterplot Smoothing regression technique,
+ including robustifying procedures against outliers. This model
+ predicts the surface rather than individual point estimates.
+
+ Initialisation Parameters:
+ frac: Fraction of the dataset to use in each local regression
+ threshold_value: Number of datetime units to use in each regression
+ threshold_units: Datetime unit which should be compatible with pandas `date_range` function
+
+ Attributes:
+ fitted: Boolean flag indicating whether the model has been fitted
+ frac: Fraction of the dataset to use in each local regression
+ threshold_value: Number of datetime units to use in each regression
+ threshold_units: Datetime unit which should be compatible with pandas `date_range` function
+ ensemble_member_to_weights: Mapping from the regression dates to their respective weightings for each data-point
+ ensemble_member_to_models: Mapping from the regression dates to their localised models
+ reg_dates: Dates at which the local time-adaptive models will be centered around
+ pred_weights: Weightings to map from the local models to the values to be inferenced
+ pred_values: Raw prediction values as generated by each of the individual local models
+ """
+
def __init__(self, frac=0.3, threshold_value=52, threshold_units='W'):
self.fitted = False
self.frac = frac
self.threshold_value = threshold_value
self.threshold_units = threshold_units
+
def fit(self, x, y, dt_idx=None, reg_dates=None, lowess_kwargs={}, **fit_kwargs):
+ """
+ Calculation of the local regression coefficients for each of the
+ LOWESS models across the dataset provided. This is a time-adaptive
+ ensembled version of the `Lowess` model.
+
+ Parameters:
+ x: Values for the independent variable
+ y: Values for the dependent variable
+ dt_idx: Datetime index, if not provided the index of the x and y series will be used
+ reg_dates: Dates at which the local time-adaptive models will be centered around
+ lowess_kwargs: Additional arguments to be passed at model initialisation
+ reg_anchors: Locations at which to center the local regressions
+ num_fits: Number of locations at which to carry out a local regression
+ external_weights: Further weighting for the specific regression
+ robust_weights: Robustifying weights to remove the influence of outliers
+ robust_iters: Number of robustifying iterations to carry out
+ """
+
x, y, dt_idx, reg_dates = process_smooth_dates_fit_inputs(x, y, dt_idx, reg_dates)
self.ensemble_member_to_weights = construct_dt_weights(dt_idx, reg_dates,
threshold_value=self.threshold_value,
@@ -470,7 +624,20 @@ def fit(self, x, y, dt_idx=None, reg_dates=None, lowess_kwargs={}, **fit_kwargs)
return
+
def predict(self, x_pred=np.linspace(8, 60, 53), dt_pred=None, return_df=True):
+ """
+ Inference using the design matrix from the time-adaptive LOWESS fits
+
+ Parameters:
+ x_pred: Independent variable locations for the time-adaptive LOWESS inference
+ dt_pred: Date locations for the time-adaptive LOWESS inference
+ return_df: Flag specifying whether to return a dataframe or numpy matrix
+
+ Returns:
+ df_pred/y_pred: Estimated surface of the time-adaptive the LOWESS fit
+ """
+
if dt_pred is None:
dt_pred = self.reg_dates
@@ -480,7 +647,10 @@ def predict(self, x_pred=np.linspace(8, 60, 53), dt_pred=None, return_df=True):
self.ensemble_member_to_preds = get_ensemble_preds(self.ensemble_member_to_models, x_pred=x_pred)
self.pred_weights = np.array(list(construct_dt_weights(dt_pred, self.reg_dates).values()))
- self.pred_weights = self.pred_weights/self.pred_weights.sum(axis=0)
+
+ with np.errstate(divide='ignore', invalid='ignore'):
+ self.pred_weights = self.pred_weights/self.pred_weights.sum(axis=0)
+
self.pred_values = np.array(list(self.ensemble_member_to_preds.values()))
y_pred = np.dot(self.pred_weights.T, self.pred_values)
@@ -489,4 +659,116 @@ def predict(self, x_pred=np.linspace(8, 60, 53), dt_pred=None, return_df=True):
df_pred = pd.DataFrame(y_pred, index=dt_pred, columns=x_pred).T
return df_pred
else:
- return y_pred
\ No newline at end of file
+ return y_pred
+
+# Cell
+def construct_pred_ts(s, df_pred, rounding_dec=1):
+ """Uses the time-adaptive LOWESS surface to generate time-series prediction"""
+ vals = []
+
+ for dt_idx, val in track(s.iteritems(), total=s.size):
+ vals += [df_pred.loc[round(val, rounding_dec), dt_idx.strftime('%Y-%m-%d')]]
+
+ s_pred_ts = pd.Series(vals, index=s.index)
+
+ return s_pred_ts
+
+class LowessDates(BaseEstimator, RegressorMixin):
+ """
+ This class provides a time-adaptive extension of the classical
+ Locally Weighted Scatterplot Smoothing regression technique,
+ including robustifying procedures against outliers.
+
+ Initialisation Parameters:
+ frac: Fraction of the dataset to use in each local regression
+ threshold_value: Number of datetime units to use in each regression
+ threshold_units: Datetime unit which should be compatible with pandas `date_range` function
+
+ Attributes:
+ fitted: Boolean flag indicating whether the model has been fitted
+ frac: Fraction of the dataset to use in each local regression
+ threshold_value: Number of datetime units to use in each regression
+ threshold_units: Datetime unit which should be compatible with pandas `date_range` function
+ ensemble_member_to_weights: Mapping from the regression dates to their respective weightings for each data-point
+ ensemble_member_to_models: Mapping from the regression dates to their localised models
+ reg_dates: Dates at which the local time-adaptive models will be centered around
+ ensemble_member_to_preds: Mapping from the regression dates to their predictions
+ reg_weights: Mapping from the prediction values to the weighting of each time-adaptive model
+ reg_values: Predictions from each regression
+ df_reg: A DataFrame of the time-adaptive surfce regression
+ """
+
+ def __init__(self, frac=0.3, threshold_value=52, threshold_units='W', pred_reg_dates=None):
+ self.fitted = False
+ self.frac = frac
+ self.threshold_value = threshold_value
+ self.threshold_units = threshold_units
+ self.pred_reg_dates = pred_reg_dates
+
+
+ def fit(self, x, y, dt_idx=None, reg_dates=None, lowess_kwargs={}, **fit_kwargs):
+ """
+ Calculation of the local regression coefficients for each of the
+ LOWESS models across the dataset provided. This is a time-adaptive
+ ensembled version of the `Lowess` model.
+
+ Parameters:
+ x: Values for the independent variable
+ y: Values for the dependent variable
+ dt_idx: Datetime index, if not provided the index of the x and y series will be used
+ reg_dates: Dates at which the local time-adaptive models will be centered around
+ lowess_kwargs: Additional arguments to be passed at model initialisation
+ reg_anchors: Locations at which to center the local regressions
+ num_fits: Number of locations at which to carry out a local regression
+ external_weights: Further weighting for the specific regression
+ robust_weights: Robustifying weights to remove the influence of outliers
+ robust_iters: Number of robustifying iterations to carry out
+ """
+
+ x, y, dt_idx, reg_dates = process_smooth_dates_fit_inputs(x, y, dt_idx, reg_dates)
+ self.ensemble_member_to_weights = construct_dt_weights(dt_idx, reg_dates,
+ threshold_value=self.threshold_value,
+ threshold_units=self.threshold_units)
+
+ self.ensemble_member_to_models = fit_external_weighted_ensemble(x, y, self.ensemble_member_to_weights, lowess_kwargs=lowess_kwargs, frac=self.frac, **fit_kwargs)
+
+ self.reg_dates = reg_dates
+ self.fitted = True
+
+ return
+
+
+ def predict(self, x_pred, reg_x=None, reg_dates=None, return_df=True, rounding_dec=1):
+ """
+ Inference using the design matrix from the time-adaptive LOWESS fits
+
+ Parameters:
+ x_pred: Locations for the time-adaptive LOWESS inference
+
+ Returns:
+ y_pred: Estimated values using the time-adaptive LOWESS fit
+ """
+
+ reg_dates = self.pred_reg_dates
+
+ if reg_x is None:
+ reg_x = np.round(np.arange(np.floor(x_pred.min())-5, np.ceil(x_pred.max())+5, 1/(10**rounding_dec)), rounding_dec)
+ x_pred = x_pred.round(rounding_dec)
+
+ if isinstance(reg_x, pd.Series):
+ reg_x = reg_x.values
+
+ # Fitting the smoothed regression
+ self.ensemble_member_to_preds = get_ensemble_preds(self.ensemble_member_to_models, x_pred=reg_x)
+
+ self.reg_weights = np.array(list(construct_dt_weights(reg_dates, self.reg_dates).values()))
+ self.reg_weights = self.reg_weights/self.reg_weights.sum(axis=0)
+ self.reg_values = np.array(list(self.ensemble_member_to_preds.values()))
+
+ y_reg = np.dot(self.reg_weights.T, self.reg_values)
+ self.df_reg = pd.DataFrame(y_reg, index=reg_dates.strftime('%Y-%m-%d'), columns=reg_x).T
+
+ # Making the prediction
+ s_pred_ts = construct_pred_ts(x_pred, self.df_reg, rounding_dec=rounding_dec)
+
+ return s_pred_ts
\ No newline at end of file
diff --git a/moepy/moe.py b/moepy/moe.py
index ad1eafb..30dea0d 100644
--- a/moepy/moe.py
+++ b/moepy/moe.py
@@ -19,7 +19,6 @@
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
-import FEAutils as hlp
from ipypb import track
from IPython.display import JSON
@@ -28,6 +27,7 @@
# Cell
def construct_dispatchable_lims_df(s_dispatchable, rolling_w=3, daily_quantiles=[0.001, 0.999]):
+ """Identifies the rolling limits to be used in masking"""
df_dispatchable_lims = (s_dispatchable
.resample('1d')
.quantile(daily_quantiles)
@@ -44,6 +44,7 @@ def construct_dispatchable_lims_df(s_dispatchable, rolling_w=3, daily_quantiles=
return df_dispatchable_lims
def construct_pred_mask_df(df_pred, df_dispatchable_lims):
+ """Constructs a DataFrame mask for the prediction"""
df_pred = df_pred[df_dispatchable_lims.index]
df_pred_mask = pd.DataFrame(dict(zip(df_pred.columns, [df_pred.index]*df_pred.shape[1])), index=df_pred.index)
df_pred_mask = (df_pred_mask > df_dispatchable_lims.iloc[:, 0].values) & (df_pred_mask < df_dispatchable_lims.iloc[:, 1].values)
@@ -55,6 +56,7 @@ def construct_pred_mask_df(df_pred, df_dispatchable_lims):
# Cell
class AxTransformer:
+ """Helper class for cleaning axis tick locations and labels"""
def __init__(self, datetime_vals=False):
self.datetime_vals = datetime_vals
self.lr = linear_model.LinearRegression()
@@ -89,6 +91,7 @@ def transform(self, tick_vals):
return tick_locs
def set_ticks(ax, tick_locs, tick_labels=None, axis='y'):
+ """Sets ticks at standard numerical locations"""
if tick_labels is None:
tick_labels = tick_locs
ax_transformer = AxTransformer()
@@ -102,6 +105,7 @@ def set_ticks(ax, tick_locs, tick_labels=None, axis='y'):
return ax
def set_date_ticks(ax, start_date, end_date, axis='y', date_format='%Y-%m-%d', **date_range_kwargs):
+ """Sets ticks at datetime locations"""
dt_rng = pd.date_range(start_date, end_date, **date_range_kwargs)
ax_transformer = AxTransformer(datetime_vals=True)
@@ -116,6 +120,7 @@ def set_date_ticks(ax, start_date, end_date, axis='y', date_format='%Y-%m-%d', *
# Cell
def construct_df_pred(model_fp, x_pred=np.linspace(-2, 61, 631), dt_pred=pd.date_range('2009-01-01', '2020-12-31', freq='1D')):
+ """Constructs the prediction surface for the specified pre-fitted model"""
smooth_dates = pickle.load(open(model_fp, 'rb'))
df_pred = smooth_dates.predict(x_pred=x_pred, dt_pred=dt_pred)
df_pred.index = np.round(df_pred.index, 1)
@@ -124,6 +129,7 @@ def construct_df_pred(model_fp, x_pred=np.linspace(-2, 61, 631), dt_pred=pd.date
# Cell
def construct_pred_ts(s, df_pred):
+ """Uses the time-adaptive LOWESS surface to generate time-series prediction"""
s_pred_ts = pd.Series(index=s.index, dtype='float64')
for dt_idx, val in track(s.iteritems(), total=s.size):
@@ -133,6 +139,7 @@ def construct_pred_ts(s, df_pred):
# Cell
def calc_error_metrics(s_err, max_err_quantile=1):
+ """Calculates several error metrics using the passed error series"""
if s_err.isnull().sum() > 0:
s_err = s_err.dropna()
@@ -149,6 +156,7 @@ def calc_error_metrics(s_err, max_err_quantile=1):
# Cell
def get_model_pred_ts(s, model_fp, s_demand=None, x_pred=np.linspace(-2, 61, 631), dt_pred=pd.date_range('2009-01-01', '2020-12-31', freq='1D')):
+ """Constructs the time-series prediction for the specified pre-fitted model"""
df_pred = construct_df_pred(model_fp, x_pred=x_pred, dt_pred=dt_pred)
s_cleaned = s.dropna().loc[df_pred.columns.min():df_pred.columns.max()+pd.Timedelta(hours=23, minutes=30)]
s_pred_ts = construct_pred_ts(s_cleaned, df_pred)
@@ -162,6 +170,7 @@ def get_model_pred_ts(s, model_fp, s_demand=None, x_pred=np.linspace(-2, 61, 631
# Cell
def weighted_mean_s(s, s_weight=None, dt_rng=pd.date_range('2009-12-01', '2021-01-01', freq='W'), end_dt_delta_days=7):
+ """Calculates the weighted average of a series"""
capture_prices = dict()
for start_dt in dt_rng:
diff --git a/moepy/retrieval.py b/moepy/retrieval.py
index f9ec020..9101998 100644
--- a/moepy/retrieval.py
+++ b/moepy/retrieval.py
@@ -2,7 +2,7 @@
__all__ = ['query_API', 'dict_col_2_cols', 'clean_nested_dict_cols', 'set_dt_idx', 'create_df_dt_rng', 'clean_df_dts',
'retrieve_stream_df', 'check_streams', 'retrieve_streams_df', 'parse_A44_response', 'retreive_DAM_prices',
- 'parse_A75_response', 'retreive_production']
+ 'parse_A75_response', 'retrieve_production']
# Cell
import json
@@ -23,11 +23,11 @@ def query_API(start_date:str, end_date:str, stream:str, time_group='30m'):
"""
'Query API' makes the call to Electric Insights and returns the JSON response
- Arguments:
- * start_date - Start date for data given as a string in the form '%Y-%m-%d'
- * end_date - End date for data given as a string in the form '%Y-%m-%d'
- * stream - One of 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions'
- * time_group - One of '30m', '1h', '1d' or '7d'. The default is '30m'
+ Parameters:
+ start_date: Start date for data given as a string in the form '%Y-%m-%d'
+ end_date: End date for data given as a string in the form '%Y-%m-%d'
+ stream: One of 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions'
+ time_group: One of '30m', '1h', '1d' or '7d'. The default is '30m'
"""
# Checking stream is an EI endpoint
@@ -68,6 +68,7 @@ def dict_col_2_cols(df:pd.DataFrame, value_col='value'):
# Cell
def clean_nested_dict_cols(df):
+ """Unpacks columns contining nested dictionaries"""
# Calculating columns that are still dictionaries
s_types = df.iloc[0].apply(lambda val: type(val))
cols_with_dicts = s_types[s_types == dict].index
@@ -122,6 +123,7 @@ def create_df_dt_rng(start_date, end_date, freq='30T', tz='Europe/London', dt_st
return df_dt_rng
def clean_df_dts(df):
+ """Cleans the datetime index of the passed DataFrame"""
df = set_dt_idx(df)
df = df[~df.index.duplicated()]
@@ -135,14 +137,14 @@ def clean_df_dts(df):
# Cell
def retrieve_stream_df(start_date:str, end_date:str, stream:str, time_group='30m', renaming_dict={}):
"""
- `retrieve_stream_df` makes the call to Electric Insights and parses the response into a dataframe which is returned
-
- Arguments:
- * start_date - Start date for data given as a string in the form '%Y-%m-%d'
- * end_date - End date for data given as a string in the form '%Y-%m-%d'
- * stream - One of 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions'
- * time_group - One of '30m', '1h', '1d' or '7d'. The default is '30m'
- * renaming_dict - Mapping from old to new column names
+ Makes the call to Electric Insights and parses the response into a dataframe which is returned
+
+ Parameters:
+ start_date: Start date for data given as a string in the form '%Y-%m-%d'
+ end_date: End date for data given as a string in the form '%Y-%m-%d'
+ stream: One of 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions'
+ time_group: One of '30m', '1h', '1d' or '7d'. The default is '30m'
+ renaming_dict: Mapping from old to new column names
"""
# Calling data and parsing into dataframe
@@ -192,13 +194,13 @@ def check_streams(streams='*'):
# Cell
def retrieve_streams_df(start_date:str, end_date:str, streams='*', time_group='30m', renaming_dict={}):
"""
- 'Call Streams' makes the calls to Electric Insights for the given streams and parses the responses into a dataframe which is returned
+ Makes the calls to Electric Insights for the given streams and parses the responses into a dataframe which is returned
- Arguments:
- * start_date - Start date for data given as a string in the form '%Y-%m-%d'
- * end_date - End date for data given as a string in the form '%Y-%m-%d'
- * streams - Contains 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions', or is given as all, '*'
- * time_group - One of '30m', '1h', '1d' or '7d'. The default is '30m'
+ Parameters:
+ start_date: Start date for data given as a string in the form '%Y-%m-%d'
+ end_date: End date for data given as a string in the form '%Y-%m-%d'
+ streams: Contains 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions', or is given as all, '*'
+ time_group: One of '30m', '1h', '1d' or '7d'. The default is '30m'
"""
df = pd.DataFrame()
@@ -212,6 +214,7 @@ def retrieve_streams_df(start_date:str, end_date:str, streams='*', time_group='3
# Cell
def parse_A44_response(r, freq='H', tz='UTC'):
+ """Extracts the price time-series"""
s_price = pd.Series(dtype=float)
parsed_r = xmltodict.parse(r.text)
@@ -227,6 +230,7 @@ def parse_A44_response(r, freq='H', tz='UTC'):
# Cell
def retreive_DAM_prices(dt_pairs, domain='10Y1001A1001A63L'):
+ """Retrieves and collates the day-ahead prices for the specified date ranges"""
params = {
'documentType': 'A44',
'in_Domain': domain,
@@ -250,7 +254,8 @@ def retreive_DAM_prices(dt_pairs, domain='10Y1001A1001A63L'):
return s_price
# Cell
-def parse_A75_response(r, freq='15T', tz='UTC'):
+def parse_A75_response(r, freq='15T', tz='UTC', warn_on_failure=False):
+ """Extracts the production data by fuel-type from the JSON response"""
psr_code_to_type = {
'A03': 'Mixed',
'A04': 'Generation',
@@ -291,7 +296,7 @@ def parse_A75_response(r, freq='15T', tz='UTC'):
df_production = pd.DataFrame(dtype=float, columns=columns, index=index)
- for timeseries in track(parsed_r['GL_MarketDocument']['TimeSeries']):
+ for timeseries in parsed_r['GL_MarketDocument']['TimeSeries']:
try:
psr_type = timeseries['MktPSRType']['psrType']
dt_rng = pd.date_range(timeseries['Period']['timeInterval']['start'], timeseries['Period']['timeInterval']['end'], freq=freq, tz=tz)[:-1]
@@ -302,7 +307,8 @@ def parse_A75_response(r, freq='15T', tz='UTC'):
df_production[psr_type] = s_psr_type
except:
- warn(f"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}")
+ if warn_on_failure == True:
+ warn(f"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}")
assert df_production.index.duplicated().sum() == 0, 'There are duplicate date indexes'
@@ -311,7 +317,8 @@ def parse_A75_response(r, freq='15T', tz='UTC'):
return df_production
-def retreive_production(dt_pairs, domain='10Y1001A1001A63L'):
+def retrieve_production(dt_pairs, domain='10Y1001A1001A63L', warn_on_failure=False):
+ """Retrieves and collates the production data for the specified date ranges"""
params = {
'documentType': 'A75',
'processType': 'A16',
@@ -327,9 +334,10 @@ def retreive_production(dt_pairs, domain='10Y1001A1001A63L'):
try:
r = client._base_request(params=params, start=start, end=end)
- df_production_dt_rng = parse_A75_response(r)
+ df_production_dt_rng = parse_A75_response(r, warn_on_failure=warn_on_failure)
df_production = df_production.append(df_production_dt_rng)
except:
- warn(f"{start.strftime('%Y-%m-%d')} - {end.strftime('%Y-%m-%d')} failed")
+ if warn_on_failure == True:
+ warn(f"{start.strftime('%Y-%m-%d')} - {end.strftime('%Y-%m-%d')} failed")
return df_production
\ No newline at end of file
diff --git a/moepy/surface.py b/moepy/surface.py
index 99308d8..70fe359 100644
--- a/moepy/surface.py
+++ b/moepy/surface.py
@@ -1,4 +1,4 @@
-# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/04-surface-estimation.ipynb (unless otherwise specified).
+# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/04-price-surface-estimation.ipynb (unless otherwise specified).
__all__ = ['PicklableFunction', 'get_fit_kwarg_sets', 'fit_models']
@@ -22,6 +22,7 @@
import marshal
class PicklableFunction:
+ """Provides a wrapper to ensure functions can be pickled"""
def __init__(self, fun):
self._fun = fun
@@ -44,6 +45,7 @@ def __setstate__(self, state):
return
def get_fit_kwarg_sets(qs=np.linspace(0.1, 0.9, 9)):
+ """Helper to generate kwargs for the `fit` method of `Lowess`"""
fit_kwarg_sets = [
# quantile lowess
{
@@ -60,6 +62,7 @@ def get_fit_kwarg_sets(qs=np.linspace(0.1, 0.9, 9)):
# Cell
def fit_models(model_definitions, models_dir):
+ """Fits LOWESS variants using the specified model definitions"""
for model_parent_name, model_spec in model_definitions.items():
for fit_kwarg_set in track(model_spec['fit_kwarg_sets'], label=model_parent_name):
run_name = fit_kwarg_set.pop('name')
diff --git a/nbs/01-retrieval.ipynb b/nbs/01-retrieval.ipynb
index 6105b52..ac109b0 100644
--- a/nbs/01-retrieval.ipynb
+++ b/nbs/01-retrieval.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
@@ -13,7 +13,14 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# Data Retrieval\n",
+ "# Data Retrieval"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "This notebook outlines the retrieval of the fuel generation and price data required for the merit-order-effect analyses.\n",
"\n",
"
\n",
"\n",
@@ -22,7 +29,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@@ -43,7 +50,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -70,7 +77,7 @@
},
{
"cell_type": "code",
- "execution_count": 23,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -79,11 +86,11 @@
" \"\"\"\n",
" 'Query API' makes the call to Electric Insights and returns the JSON response\n",
"\n",
- " Arguments:\n",
- " * start_date - Start date for data given as a string in the form '%Y-%m-%d'\n",
- " * end_date - End date for data given as a string in the form '%Y-%m-%d'\n",
- " * stream - One of 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions'\n",
- " * time_group - One of '30m', '1h', '1d' or '7d'. The default is '30m'\n",
+ " Parameters:\n",
+ " start_date: Start date for data given as a string in the form '%Y-%m-%d'\n",
+ " end_date: End date for data given as a string in the form '%Y-%m-%d'\n",
+ " stream: One of 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions'\n",
+ " time_group: One of '30m', '1h', '1d' or '7d'. The default is '30m'\n",
" \"\"\"\n",
"\n",
" # Checking stream is an EI endpoint\n",
@@ -108,7 +115,7 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -103801,7 +103808,7 @@
""
]
},
- "execution_count": 24,
+ "execution_count": 5,
"metadata": {
"application/json": {
"expanded": false,
@@ -103832,7 +103839,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
@@ -103925,7 +103932,7 @@
"4 {'nuclear': 6.827, 'biomass': 1.081, 'coal': 0... "
]
},
- "execution_count": 25,
+ "execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@@ -103947,7 +103954,7 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@@ -103971,7 +103978,7 @@
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": 8,
"metadata": {},
"outputs": [
{
@@ -104112,7 +104119,7 @@
"2 {'french': 1.504, 'dutch': 0.588, 'irish': -0.... "
]
},
- "execution_count": 27,
+ "execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
@@ -104134,12 +104141,13 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"#exports\n",
"def clean_nested_dict_cols(df):\n",
+ " \"\"\"Unpacks columns contining nested dictionaries\"\"\"\n",
" # Calculating columns that are still dictionaries\n",
" s_types = df.iloc[0].apply(lambda val: type(val))\n",
" cols_with_dicts = s_types[s_types == dict].index\n",
@@ -104158,7 +104166,7 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
@@ -104341,7 +104349,7 @@
"4 0.0 0.678 1.504 0.0 0.0 -0.910 "
]
},
- "execution_count": 29,
+ "execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
@@ -104363,7 +104371,7 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
@@ -104406,6 +104414,7 @@
" return df_dt_rng\n",
"\n",
"def clean_df_dts(df):\n",
+ " \"\"\"Cleans the datetime index of the passed DataFrame\"\"\"\n",
" df = set_dt_idx(df)\n",
" df = df[~df.index.duplicated()] \n",
"\n",
@@ -104419,7 +104428,7 @@
},
{
"cell_type": "code",
- "execution_count": 31,
+ "execution_count": 12,
"metadata": {},
"outputs": [
{
@@ -104628,7 +104637,7 @@
"2019-01-01 02:00:00+00:00 -0.910 5 "
]
},
- "execution_count": 31,
+ "execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
@@ -104650,21 +104659,21 @@
},
{
"cell_type": "code",
- "execution_count": 32,
+ "execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"#exports\n",
"def retrieve_stream_df(start_date:str, end_date:str, stream:str, time_group='30m', renaming_dict={}):\n",
" \"\"\"\n",
- " `retrieve_stream_df` makes the call to Electric Insights and parses the response into a dataframe which is returned\n",
+ " Makes the call to Electric Insights and parses the response into a dataframe which is returned\n",
"\n",
- " Arguments:\n",
- " * start_date - Start date for data given as a string in the form '%Y-%m-%d'\n",
- " * end_date - End date for data given as a string in the form '%Y-%m-%d'\n",
- " * stream - One of 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions'\n",
- " * time_group - One of '30m', '1h', '1d' or '7d'. The default is '30m'\n",
- " * renaming_dict - Mapping from old to new column names\n",
+ " Parameters:\n",
+ " start_date: Start date for data given as a string in the form '%Y-%m-%d'\n",
+ " end_date: End date for data given as a string in the form '%Y-%m-%d'\n",
+ " stream: One of 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions'\n",
+ " time_group: One of '30m', '1h', '1d' or '7d'. The default is '30m'\n",
+ " renaming_dict: Mapping from old to new column names\n",
" \"\"\"\n",
"\n",
" # Calling data and parsing into dataframe\n",
@@ -104691,7 +104700,7 @@
},
{
"cell_type": "code",
- "execution_count": 35,
+ "execution_count": 14,
"metadata": {},
"outputs": [
{
@@ -104900,7 +104909,7 @@
"2019-01-01 02:00:00+00:00 0.0 -0.910 5 "
]
},
- "execution_count": 35,
+ "execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
@@ -104935,7 +104944,7 @@
},
{
"cell_type": "code",
- "execution_count": 65,
+ "execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
@@ -104965,7 +104974,7 @@
},
{
"cell_type": "code",
- "execution_count": 66,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
@@ -104974,7 +104983,7 @@
"['prices_ahead', 'prices', 'temperatures', 'emissions', 'generation-mix']"
]
},
- "execution_count": 66,
+ "execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
@@ -104996,7 +105005,7 @@
},
{
"cell_type": "code",
- "execution_count": 67,
+ "execution_count": 17,
"metadata": {},
"outputs": [
{
@@ -105005,7 +105014,7 @@
"['prices', 'emissions']"
]
},
- "execution_count": 67,
+ "execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
@@ -105027,7 +105036,7 @@
},
{
"cell_type": "code",
- "execution_count": 68,
+ "execution_count": 18,
"metadata": {},
"outputs": [
{
@@ -105059,20 +105068,20 @@
},
{
"cell_type": "code",
- "execution_count": 71,
+ "execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"#exports\n",
"def retrieve_streams_df(start_date:str, end_date:str, streams='*', time_group='30m', renaming_dict={}):\n",
" \"\"\"\n",
- " 'Call Streams' makes the calls to Electric Insights for the given streams and parses the responses into a dataframe which is returned\n",
+ " Makes the calls to Electric Insights for the given streams and parses the responses into a dataframe which is returned\n",
"\n",
- " Arguments:\n",
- " * start_date - Start date for data given as a string in the form '%Y-%m-%d'\n",
- " * end_date - End date for data given as a string in the form '%Y-%m-%d'\n",
- " * streams - Contains 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions', or is given as all, '*'\n",
- " * time_group - One of '30m', '1h', '1d' or '7d'. The default is '30m'\n",
+ " Parameters:\n",
+ " start_date: Start date for data given as a string in the form '%Y-%m-%d'\n",
+ " end_date: End date for data given as a string in the form '%Y-%m-%d'\n",
+ " streams: Contains 'prices_ahead', 'prices_ahead', 'prices', 'temperatures' or 'emissions', or is given as all, '*'\n",
+ " time_group: One of '30m', '1h', '1d' or '7d'. The default is '30m'\n",
" \"\"\"\n",
"\n",
" df = pd.DataFrame()\n",
@@ -105087,66 +105096,9 @@
},
{
"cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "streams = '*'\n",
- "renaming_dict = {\n",
- " 'pumpedStorage' : 'pumped_storage',\n",
- " 'northernIreland' : 'northern_ireland',\n",
- " 'windOnshore': 'wind_onshore',\n",
- " 'windOffshore': 'wind_offshore',\n",
- " 'prices_ahead' : 'day_ahead_price',\n",
- " 'prices' : 'imbalance_price',\n",
- " 'temperatures' : 'temperature',\n",
- " 'totalInGperkWh' : 'gCO2_per_kWh',\n",
- " 'totalInTperh' : 'TCO2_per_h'\n",
- "}\n",
- "\n",
- "df = retrieve_streams_df(start_date, end_date, streams, renaming_dict=renaming_dict)\n",
- "\n",
- "df.head()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "
\n",
- "\n",
- "Now we're ready to retrieve all of the streams in one, which we'll do for all years that data is available"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 85,
+ "execution_count": 20,
"metadata": {},
"outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- "
\n",
- "
100%\n",
- "
12/12\n",
- "
[12:03<01:04, 60.21s/it] "
- ],
- "text/plain": [
- "\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 12/12 [12:03<01:04, 60.21s/it]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Wall time: 12min 2s\n"
- ]
- },
{
"data": {
"text/html": [
@@ -105217,124 +105169,124 @@
" \n",
" \n",
" \n",
- " 2009-01-01 00:00:00+00:00 | \n",
- " 58.05 | \n",
+ " 2019-01-01 00:00:00+00:00 | \n",
+ " 48.81 | \n",
" 1 | \n",
- " 74.74 | \n",
- " 74.74 | \n",
- " -0.6 | \n",
- " 21278.0 | \n",
- " 555.0 | \n",
- " 6.973 | \n",
+ " 15.0 | \n",
+ " 15.0 | \n",
+ " 9.1 | \n",
+ " 2287.010 | \n",
+ " 83.662935 | \n",
+ " 6.924 | \n",
+ " 1.116 | \n",
" 0 | \n",
- " 17.650 | \n",
" ... | \n",
- " 38.329 | \n",
- " -0.404 | \n",
- " None | \n",
- " None | \n",
- " 0.0 | \n",
+ " 27.336 | \n",
+ " 0.000 | \n",
+ " 8.054581 | \n",
+ " 3.141711 | \n",
" 0.0 | \n",
- " 1.977 | \n",
+ " 0.182 | \n",
+ " 1.552 | \n",
" 0.0 | \n",
" 0.0 | \n",
- " -0.161 | \n",
+ " -0.702 | \n",
"
\n",
" \n",
- " 2009-01-01 00:30:00+00:00 | \n",
- " 56.33 | \n",
+ " 2019-01-01 00:30:00+00:00 | \n",
+ " 50.24 | \n",
" 2 | \n",
- " 74.89 | \n",
- " 74.89 | \n",
- " -0.6 | \n",
- " 21442.0 | \n",
- " 558.0 | \n",
- " 6.968 | \n",
+ " 15.0 | \n",
+ " 15.0 | \n",
+ " 9.1 | \n",
+ " 2467.906 | \n",
+ " 89.023375 | \n",
+ " 6.838 | \n",
+ " 1.103 | \n",
" 0 | \n",
- " 17.770 | \n",
" ... | \n",
- " 38.461 | \n",
- " -0.527 | \n",
- " None | \n",
- " None | \n",
- " 0.0 | \n",
+ " 27.722 | \n",
+ " 0.024 | \n",
+ " 7.860487 | \n",
+ " 3.253887 | \n",
" 0.0 | \n",
- " 1.977 | \n",
+ " 0.196 | \n",
+ " 1.554 | \n",
" 0.0 | \n",
" 0.0 | \n",
- " -0.160 | \n",
+ " -0.696 | \n",
"
\n",
" \n",
- " 2009-01-01 01:00:00+00:00 | \n",
- " 52.98 | \n",
+ " 2019-01-01 01:00:00+00:00 | \n",
+ " 41.90 | \n",
" 3 | \n",
- " 76.41 | \n",
- " 76.41 | \n",
- " -0.6 | \n",
- " 21614.0 | \n",
- " 569.0 | \n",
- " 6.970 | \n",
+ " 16.0 | \n",
+ " 16.0 | \n",
+ " 9.1 | \n",
+ " 2411.834 | \n",
+ " 87.888419 | \n",
+ " 6.834 | \n",
+ " 1.090 | \n",
" 0 | \n",
- " 18.070 | \n",
" ... | \n",
- " 37.986 | \n",
- " -1.018 | \n",
- " None | \n",
- " None | \n",
- " 0.0 | \n",
+ " 27.442 | \n",
+ " 0.000 | \n",
+ " 7.879198 | \n",
+ " 3.340851 | \n",
" 0.0 | \n",
- " 1.977 | \n",
+ " 0.588 | \n",
+ " 1.504 | \n",
" 0.0 | \n",
" 0.0 | \n",
- " -0.160 | \n",
+ " -0.722 | \n",
"
\n",
" \n",
- " 2009-01-01 01:30:00+00:00 | \n",
- " 50.39 | \n",
+ " 2019-01-01 01:30:00+00:00 | \n",
+ " 39.32 | \n",
" 4 | \n",
- " 37.73 | \n",
- " 37.73 | \n",
- " -0.6 | \n",
- " 21320.0 | \n",
- " 578.0 | \n",
- " 6.969 | \n",
+ " 16.0 | \n",
+ " 16.0 | \n",
+ " 9.1 | \n",
+ " 2119.532 | \n",
+ " 80.072988 | \n",
+ " 6.830 | \n",
+ " 1.085 | \n",
" 0 | \n",
- " 18.022 | \n",
" ... | \n",
- " 36.864 | \n",
- " -1.269 | \n",
- " None | \n",
- " None | \n",
- " 0.0 | \n",
+ " 26.470 | \n",
+ " 0.000 | \n",
+ " 7.708874 | \n",
+ " 3.213702 | \n",
" 0.0 | \n",
- " 1.746 | \n",
+ " 0.600 | \n",
+ " 1.504 | \n",
" 0.0 | \n",
" 0.0 | \n",
- " -0.160 | \n",
+ " -0.770 | \n",
"
\n",
" \n",
- " 2009-01-01 02:00:00+00:00 | \n",
- " 48.70 | \n",
+ " 2019-01-01 02:00:00+00:00 | \n",
+ " 34.09 | \n",
" 5 | \n",
- " 59.00 | \n",
- " 59.00 | \n",
- " -0.6 | \n",
- " 21160.0 | \n",
- " 585.0 | \n",
- " 6.960 | \n",
+ " 16.0 | \n",
+ " 16.0 | \n",
+ " 9.1 | \n",
+ " 2069.840 | \n",
+ " 79.016606 | \n",
+ " 6.827 | \n",
+ " 1.081 | \n",
" 0 | \n",
- " 17.998 | \n",
" ... | \n",
- " 36.180 | \n",
- " -1.566 | \n",
- " None | \n",
- " None | \n",
- " 0.0 | \n",
+ " 26.195 | \n",
+ " 0.000 | \n",
+ " 7.479429 | \n",
+ " 3.122706 | \n",
" 0.0 | \n",
- " 1.730 | \n",
+ " 0.678 | \n",
+ " 1.504 | \n",
" 0.0 | \n",
" 0.0 | \n",
- " -0.160 | \n",
+ " -0.910 | \n",
"
\n",
" \n",
"\n",
@@ -105344,55 +105296,53 @@
"text/plain": [
" day_ahead_price SP imbalance_price valueSum \\\n",
"local_datetime \n",
- "2009-01-01 00:00:00+00:00 58.05 1 74.74 74.74 \n",
- "2009-01-01 00:30:00+00:00 56.33 2 74.89 74.89 \n",
- "2009-01-01 01:00:00+00:00 52.98 3 76.41 76.41 \n",
- "2009-01-01 01:30:00+00:00 50.39 4 37.73 37.73 \n",
- "2009-01-01 02:00:00+00:00 48.70 5 59.00 59.00 \n",
+ "2019-01-01 00:00:00+00:00 48.81 1 15.0 15.0 \n",
+ "2019-01-01 00:30:00+00:00 50.24 2 15.0 15.0 \n",
+ "2019-01-01 01:00:00+00:00 41.90 3 16.0 16.0 \n",
+ "2019-01-01 01:30:00+00:00 39.32 4 16.0 16.0 \n",
+ "2019-01-01 02:00:00+00:00 34.09 5 16.0 16.0 \n",
"\n",
" temperature TCO2_per_h gCO2_per_kWh nuclear \\\n",
"local_datetime \n",
- "2009-01-01 00:00:00+00:00 -0.6 21278.0 555.0 6.973 \n",
- "2009-01-01 00:30:00+00:00 -0.6 21442.0 558.0 6.968 \n",
- "2009-01-01 01:00:00+00:00 -0.6 21614.0 569.0 6.970 \n",
- "2009-01-01 01:30:00+00:00 -0.6 21320.0 578.0 6.969 \n",
- "2009-01-01 02:00:00+00:00 -0.6 21160.0 585.0 6.960 \n",
+ "2019-01-01 00:00:00+00:00 9.1 2287.010 83.662935 6.924 \n",
+ "2019-01-01 00:30:00+00:00 9.1 2467.906 89.023375 6.838 \n",
+ "2019-01-01 01:00:00+00:00 9.1 2411.834 87.888419 6.834 \n",
+ "2019-01-01 01:30:00+00:00 9.1 2119.532 80.072988 6.830 \n",
+ "2019-01-01 02:00:00+00:00 9.1 2069.840 79.016606 6.827 \n",
"\n",
- " biomass coal ... demand pumped_storage \\\n",
- "local_datetime ... \n",
- "2009-01-01 00:00:00+00:00 0 17.650 ... 38.329 -0.404 \n",
- "2009-01-01 00:30:00+00:00 0 17.770 ... 38.461 -0.527 \n",
- "2009-01-01 01:00:00+00:00 0 18.070 ... 37.986 -1.018 \n",
- "2009-01-01 01:30:00+00:00 0 18.022 ... 36.864 -1.269 \n",
- "2009-01-01 02:00:00+00:00 0 17.998 ... 36.180 -1.566 \n",
+ " biomass coal ... demand pumped_storage \\\n",
+ "local_datetime ... \n",
+ "2019-01-01 00:00:00+00:00 1.116 0 ... 27.336 0.000 \n",
+ "2019-01-01 00:30:00+00:00 1.103 0 ... 27.722 0.024 \n",
+ "2019-01-01 01:00:00+00:00 1.090 0 ... 27.442 0.000 \n",
+ "2019-01-01 01:30:00+00:00 1.085 0 ... 26.470 0.000 \n",
+ "2019-01-01 02:00:00+00:00 1.081 0 ... 26.195 0.000 \n",
"\n",
- " wind_onshore wind_offshore belgian dutch french \\\n",
- "local_datetime \n",
- "2009-01-01 00:00:00+00:00 None None 0.0 0.0 1.977 \n",
- "2009-01-01 00:30:00+00:00 None None 0.0 0.0 1.977 \n",
- "2009-01-01 01:00:00+00:00 None None 0.0 0.0 1.977 \n",
- "2009-01-01 01:30:00+00:00 None None 0.0 0.0 1.746 \n",
- "2009-01-01 02:00:00+00:00 None None 0.0 0.0 1.730 \n",
+ " wind_onshore wind_offshore belgian dutch french \\\n",
+ "local_datetime \n",
+ "2019-01-01 00:00:00+00:00 8.054581 3.141711 0.0 0.182 1.552 \n",
+ "2019-01-01 00:30:00+00:00 7.860487 3.253887 0.0 0.196 1.554 \n",
+ "2019-01-01 01:00:00+00:00 7.879198 3.340851 0.0 0.588 1.504 \n",
+ "2019-01-01 01:30:00+00:00 7.708874 3.213702 0.0 0.600 1.504 \n",
+ "2019-01-01 02:00:00+00:00 7.479429 3.122706 0.0 0.678 1.504 \n",
"\n",
- " ireland northern_ireland irish \n",
- "local_datetime \n",
- "2009-01-01 00:00:00+00:00 0.0 0.0 -0.161 \n",
- "2009-01-01 00:30:00+00:00 0.0 0.0 -0.160 \n",
- "2009-01-01 01:00:00+00:00 0.0 0.0 -0.160 \n",
- "2009-01-01 01:30:00+00:00 0.0 0.0 -0.160 \n",
- "2009-01-01 02:00:00+00:00 0.0 0.0 -0.160 \n",
+ " ireland northern_ireland irish \n",
+ "local_datetime \n",
+ "2019-01-01 00:00:00+00:00 0.0 0.0 -0.702 \n",
+ "2019-01-01 00:30:00+00:00 0.0 0.0 -0.696 \n",
+ "2019-01-01 01:00:00+00:00 0.0 0.0 -0.722 \n",
+ "2019-01-01 01:30:00+00:00 0.0 0.0 -0.770 \n",
+ "2019-01-01 02:00:00+00:00 0.0 0.0 -0.910 \n",
"\n",
"[5 rows x 24 columns]"
]
},
- "execution_count": 85,
+ "execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "%%time\n",
- "\n",
"streams = '*'\n",
"renaming_dict = {\n",
" 'pumpedStorage' : 'pumped_storage',\n",
@@ -105406,14 +105356,7 @@
" 'totalInTperh' : 'TCO2_per_h'\n",
"}\n",
"\n",
- "df = pd.DataFrame()\n",
- "\n",
- "for year in track(range(2009, 2021)):\n",
- " start_date = f'{year}-01-01'\n",
- " end_date = f'{year}-12-31'\n",
- " \n",
- " df_year = retrieve_streams_df(start_date, end_date, streams, renaming_dict=renaming_dict)\n",
- " df = df.append(df_year)\n",
+ "df = retrieve_streams_df(start_date, end_date, streams, renaming_dict=renaming_dict)\n",
"\n",
"df.head()"
]
@@ -105424,35 +105367,51 @@
"source": [
"
\n",
"\n",
- "We'll now save the retrieved data"
+ "Now we're ready to retrieve all of the streams in one, which we'll do for all years that data is available, then we'll save the resulting DataFrame."
]
},
{
"cell_type": "code",
- "execution_count": 86,
- "metadata": {},
- "outputs": [],
- "source": [
- "df.to_csv('../data/electric_insights.csv')"
- ]
- },
- {
- "cell_type": "markdown",
+ "execution_count": 21,
"metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Wall time: 0 ns\n"
+ ]
+ }
+ ],
"source": [
- "
\n",
+ "%%time\n",
"\n",
- "We'll also save the renaming dictionary to ensure we can fully reproduce this at a later date"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 78,
- "metadata": {},
- "outputs": [],
- "source": [
- "with open('../data/EI_rename_mapping.json', 'w') as fp:\n",
- " json.dump(renaming_dict, fp)"
+ "streams = '*'\n",
+ "renaming_dict = {\n",
+ " 'pumpedStorage' : 'pumped_storage',\n",
+ " 'northernIreland' : 'northern_ireland',\n",
+ " 'windOnshore': 'wind_onshore',\n",
+ " 'windOffshore': 'wind_offshore',\n",
+ " 'prices_ahead' : 'day_ahead_price',\n",
+ " 'prices' : 'imbalance_price',\n",
+ " 'temperatures' : 'temperature',\n",
+ " 'totalInGperkWh' : 'gCO2_per_kWh',\n",
+ " 'totalInTperh' : 'TCO2_per_h'\n",
+ "}\n",
+ "\n",
+ "retrieve_save_EI_data = False\n",
+ "\n",
+ "if retrieve_save_EI_data == True:\n",
+ " df = pd.DataFrame()\n",
+ "\n",
+ " for year in track(range(2010, 2021)):\n",
+ " start_date = f'{year}-01-01'\n",
+ " end_date = f'{year}-12-31'\n",
+ "\n",
+ " df_year = retrieve_streams_df(start_date, end_date, streams, renaming_dict=renaming_dict)\n",
+ " df = df.append(df_year)\n",
+ "\n",
+ " df.to_csv('../data/raw/electric_insights.csv')"
]
},
{
@@ -105470,7 +105429,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
@@ -105479,13 +105438,14 @@
""
]
},
- "execution_count": 3,
+ "execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def year_week_2_prod_url(year, week, data_prefix=''): \n",
+ " \"\"\"Given a specified year and week the relevant `production_url` for energy-charts is returned\"\"\"\n",
" if year < 2019:\n",
" data_prefix = 'raw_'\n",
" \n",
@@ -105514,7 +105474,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 23,
"metadata": {},
"outputs": [
{
@@ -105543,13 +105503,14 @@
"Name: value, Length: 167, dtype: float64"
]
},
- "execution_count": 4,
+ "execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def fuel_json_2_net_balance(r_json):\n",
+ " \"\"\"Extracts the balance time-series\"\"\"\n",
" if 'values' in r_json[0].keys(): # pre-2019 format\n",
" df_balance = pd.DataFrame(r_json[0]['values'])\n",
"\n",
@@ -105595,7 +105556,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
@@ -105704,13 +105665,14 @@
"2018-03-19 01:00:00+01:00 0.064 19.012 0.0 -9.522 "
]
},
- "execution_count": 5,
+ "execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def response_2_df(r):\n",
+ " \"\"\"Parses the json response to a DataFrame\"\"\"\n",
" r_json = r.json()\n",
" s_balance = fuel_json_2_net_balance(r_json)\n",
"\n",
@@ -105747,7 +105709,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 25,
"metadata": {},
"outputs": [
{
@@ -105856,13 +105818,14 @@
"2015-12-28 01:00:00+01:00 0.012 7.059 0.0 -4.139 "
]
},
- "execution_count": 6,
+ "execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def year_week_2_fuel_df(year, week):\n",
+ " \"\"\"Given a specified year and week the relevant `df_fuels` dataset for energy-charts is returned\"\"\"\n",
" production_url = year_week_2_prod_url(year, week)\n",
"\n",
" r = requests.get(production_url)\n",
@@ -105895,7 +105858,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 26,
"metadata": {},
"outputs": [
{
@@ -105905,11 +105868,11 @@
"\n",
"100%\n",
"580/583\n",
- "[03:29<00:00, 0.36s/it]"
+ "[03:16<00:00, 0.34s/it]"
],
"text/plain": [
"\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 583/583 [03:29<00:00, 0.36s/it]"
+ " [████████████████████████████████████████████████████████████] 583/583 [03:16<00:00, 0.34s/it]"
]
},
"metadata": {},
@@ -106071,7 +106034,7 @@
"[3 rows x 32 columns]"
]
},
- "execution_count": 14,
+ "execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
@@ -106103,7 +106066,7 @@
},
{
"cell_type": "code",
- "execution_count": 37,
+ "execution_count": 27,
"metadata": {},
"outputs": [
{
@@ -106250,7 +106213,7 @@
"2010-01-04 04:00:00+01:00 0.0 16.635 0.713 -0.731 "
]
},
- "execution_count": 37,
+ "execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
@@ -106282,7 +106245,7 @@
},
{
"cell_type": "code",
- "execution_count": 38,
+ "execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
@@ -106300,7 +106263,7 @@
},
{
"cell_type": "code",
- "execution_count": 39,
+ "execution_count": 29,
"metadata": {},
"outputs": [
{
@@ -106330,7 +106293,7 @@
},
{
"cell_type": "code",
- "execution_count": 40,
+ "execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
@@ -106348,12 +106311,12 @@
},
{
"cell_type": "code",
- "execution_count": 41,
+ "execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"df_fuels_clean.index.name = 'local_datetime'\n",
- "df_fuels_clean.to_csv('../data/energy_charts.csv')"
+ "df_fuels_clean.to_csv('../data/raw/energy_charts.csv')"
]
},
{
@@ -106375,16 +106338,16 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 32,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- ""
+ ""
]
},
- "execution_count": 5,
+ "execution_count": 32,
"metadata": {},
"output_type": "execute_result"
}
@@ -106398,15 +106361,17 @@
]
},
{
- "cell_type": "code",
- "execution_count": 116,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
- "source": []
+ "source": [
+ "
\n",
+ "\n",
+ "We'll then make a test request for price data from the German market"
+ ]
},
{
"cell_type": "code",
- "execution_count": 69,
+ "execution_count": 33,
"metadata": {},
"outputs": [
{
@@ -106415,7 +106380,7 @@
""
]
},
- "execution_count": 69,
+ "execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
@@ -106436,20 +106401,23 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
- "source": []
+ "source": [
+ "
\n",
+ "\n",
+ "We'll extract the price time-series from the returned JSON"
+ ]
},
{
"cell_type": "code",
- "execution_count": 88,
+ "execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"#exports\n",
"def parse_A44_response(r, freq='H', tz='UTC'):\n",
+ " \"\"\"Extracts the price time-series\"\"\"\n",
" s_price = pd.Series(dtype=float)\n",
" parsed_r = xmltodict.parse(r.text)\n",
" \n",
@@ -106466,7 +106434,7 @@
},
{
"cell_type": "code",
- "execution_count": 71,
+ "execution_count": 35,
"metadata": {},
"outputs": [
{
@@ -106477,10 +106445,10 @@
"2018-09-01 00:00:00+00:00 55.56\n",
"2018-09-01 01:00:00+00:00 54.02\n",
"2018-09-01 02:00:00+00:00 52.69\n",
- "Freq: H, dtype: object"
+ "Freq: H, dtype: float64"
]
},
- "execution_count": 71,
+ "execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
@@ -106492,18 +106460,19 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
"source": [
- "# 10Y1001A1001A63L - up to '2018-09-30' (DE-AT-LU)\n",
- "# 10Y1001A1001A82H - 2018-10-01' onwards (DE-LU)"
+ "
\n",
+ "\n",
+ "We can't query very large date ranges so we'll break up our requests into quarterly batches. \n",
+ "\n",
+ "We'll also account for the fact that the German market changes to exclude Austria after 2018-10-01 by creating two sets of date pairs."
]
},
{
"cell_type": "code",
- "execution_count": 195,
+ "execution_count": 36,
"metadata": {},
"outputs": [
{
@@ -106516,7 +106485,7 @@
" ('2016-01-01 00:00', '2016-03-31 23:55')]"
]
},
- "execution_count": 195,
+ "execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
@@ -106531,20 +106500,23 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
- "source": []
+ "source": [
+ "
\n",
+ "\n",
+ "We're now ready to create a wrapper to collate the date from each date batch"
+ ]
},
{
"cell_type": "code",
- "execution_count": 207,
+ "execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"#exports\n",
"def retreive_DAM_prices(dt_pairs, domain='10Y1001A1001A63L'):\n",
+ " \"\"\"Retrieves and collates the day-ahead prices for the specified date ranges\"\"\"\n",
" params = {\n",
" 'documentType': 'A44',\n",
" 'in_Domain': domain,\n",
@@ -106570,7 +106542,7 @@
},
{
"cell_type": "code",
- "execution_count": 93,
+ "execution_count": 38,
"metadata": {},
"outputs": [
{
@@ -106580,11 +106552,11 @@
"\n",
"100%\n",
"15/15\n",
- "[00:19<00:01, 1.27s/it]"
+ "[00:20<00:01, 1.31s/it]"
],
"text/plain": [
"\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 15/15 [00:19<00:01, 1.27s/it]"
+ " [████████████████████████████████████████████████████████████] 15/15 [00:20<00:01, 1.31s/it]"
]
},
"metadata": {},
@@ -106601,7 +106573,7 @@
"dtype: float64"
]
},
- "execution_count": 93,
+ "execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
@@ -106613,17 +106585,36 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
- "source": []
+ "source": [
+ "
\n",
+ "\n",
+ "We'll repeat this for the market excluding Austria"
+ ]
},
{
"cell_type": "code",
- "execution_count": 203,
+ "execution_count": 39,
"metadata": {},
"outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "
\n",
+ "
100%\n",
+ "
9/9\n",
+ "
[00:11<00:01, 1.25s/it] "
+ ],
+ "text/plain": [
+ "\u001b[A\u001b[2K\r",
+ " [████████████████████████████████████████████████████████████] 9/9 [00:11<00:01, 1.25s/it]"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
{
"data": {
"text/plain": [
@@ -106635,7 +106626,7 @@
"dtype: float64"
]
},
- "execution_count": 203,
+ "execution_count": 39,
"metadata": {},
"output_type": "execute_result"
}
@@ -106652,15 +106643,17 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
- "source": []
+ "source": [
+ "
\n",
+ "\n",
+ "We'll now combine and visualise the time-series"
+ ]
},
{
"cell_type": "code",
- "execution_count": 113,
+ "execution_count": 40,
"metadata": {},
"outputs": [
{
@@ -106669,7 +106662,7 @@
""
]
},
- "execution_count": 113,
+ "execution_count": 40,
"metadata": {},
"output_type": "execute_result"
},
@@ -106694,22 +106687,24 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
- "source": []
+ "source": [
+ "
\n",
+ "\n",
+ "Before moving on we'll save this series as a csv"
+ ]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 41,
"metadata": {},
"outputs": [],
"source": [
"s_price.name = 'DE_price'\n",
"s_price.index.name = 'local_datetime'\n",
"\n",
- "s_price.to_csv('../data/ENTSOE_DE_price.csv')"
+ "s_price.to_csv('../data/raw/ENTSOE_DE_price.csv')"
]
},
{
@@ -106718,17 +106713,20 @@
"source": [
"
\n",
"\n",
- "##### Generation by Fuel-Type"
+ "##### Generation by Fuel-Type\n",
+ "\n",
+ "We'll now create the functions for retrieving and parsing the fuel data"
]
},
{
"cell_type": "code",
- "execution_count": 209,
+ "execution_count": 42,
"metadata": {},
"outputs": [],
"source": [
"#exports\n",
- "def parse_A75_response(r, freq='15T', tz='UTC'):\n",
+ "def parse_A75_response(r, freq='15T', tz='UTC', warn_on_failure=False):\n",
+ " \"\"\"Extracts the production data by fuel-type from the JSON response\"\"\"\n",
" psr_code_to_type = {\n",
" 'A03': 'Mixed',\n",
" 'A04': 'Generation',\n",
@@ -106769,7 +106767,7 @@
" \n",
" df_production = pd.DataFrame(dtype=float, columns=columns, index=index)\n",
" \n",
- " for timeseries in track(parsed_r['GL_MarketDocument']['TimeSeries']):\n",
+ " for timeseries in parsed_r['GL_MarketDocument']['TimeSeries']:\n",
" try:\n",
" psr_type = timeseries['MktPSRType']['psrType']\n",
" dt_rng = pd.date_range(timeseries['Period']['timeInterval']['start'], timeseries['Period']['timeInterval']['end'], freq=freq, tz=tz)[:-1]\n",
@@ -106780,7 +106778,8 @@
" df_production[psr_type] = s_psr_type\n",
" \n",
" except:\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
+ " if warn_on_failure == True:\n",
+ " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
" \n",
" assert df_production.index.duplicated().sum() == 0, 'There are duplicate date indexes'\n",
" \n",
@@ -106789,7 +106788,8 @@
" \n",
" return df_production\n",
"\n",
- "def retreive_production(dt_pairs, domain='10Y1001A1001A63L'):\n",
+ "def retrieve_production(dt_pairs, domain='10Y1001A1001A63L', warn_on_failure=False):\n",
+ " \"\"\"Retrieves and collates the production data for the specified date ranges\"\"\"\n",
" params = {\n",
" 'documentType': 'A75',\n",
" 'processType': 'A16',\n",
@@ -106805,1180 +106805,54 @@
" try:\n",
" r = client._base_request(params=params, start=start, end=end)\n",
"\n",
- " df_production_dt_rng = parse_A75_response(r)\n",
+ " df_production_dt_rng = parse_A75_response(r, warn_on_failure=warn_on_failure)\n",
" df_production = df_production.append(df_production_dt_rng)\n",
" except:\n",
- " warn(f\"{start.strftime('%Y-%m-%d')} - {end.strftime('%Y-%m-%d')} failed\")\n",
+ " if warn_on_failure == True:\n",
+ " warn(f\"{start.strftime('%Y-%m-%d')} - {end.strftime('%Y-%m-%d')} failed\")\n",
" \n",
" return df_production"
]
},
{
"cell_type": "code",
- "execution_count": 211,
- "metadata": {
- "collapsed": true,
- "jupyter": {
- "outputs_hidden": true
- }
- },
+ "execution_count": 43,
+ "metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
- "
\n",
- "
100%\n",
- "
9/9\n",
- "
[03:07<00:16, 20.81s/it] "
- ],
- "text/plain": [
- "\u001b[A\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 9/9 [03:07<00:16, 20.81s/it]\u001b[B"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- "\n",
- "
\n",
- "
100%\n",
- "
147/147\n",
- "
[00:01<00:00, 0.00s/it] "
- ],
- "text/plain": [
- "\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 147/147 [00:01<00:00, 0.00s/it]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- ":54: UserWarning: 2018-11-25T00:00Z-2018-11-25T00:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2018-12-05T15:30Z-2018-12-05T15:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2018-12-15T05:00Z-2018-12-15T05:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2018-12-29T07:00Z-2018-12-29T07:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2018-10-14T19:15Z-2018-10-14T19:15Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2018-10-28T05:15Z-2018-10-28T05:15Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2018-12-23T03:45Z-2018-12-23T03:45Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- "
\n",
- "
100%\n",
- "
105/105\n",
- "
[00:00<00:00, 0.00s/it] "
- ],
- "text/plain": [
- "\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 105/105 [00:00<00:00, 0.00s/it]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- ":54: UserWarning: 2019-03-28T12:15Z-2019-03-28T12:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- "
\n",
- "
100%\n",
- "
87/87\n",
- "
[00:00<00:00, 0.00s/it] "
- ],
- "text/plain": [
- "\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 87/87 [00:00<00:00, 0.00s/it]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- ":54: UserWarning: 2019-04-01T21:45Z-2019-04-01T21:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-04-01T22:15Z-2019-04-01T22:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-04-12T23:15Z-2019-04-12T23:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-04-13T10:00Z-2019-04-13T10:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-04-19T07:45Z-2019-04-19T07:45Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-04-05T07:45Z-2019-04-05T07:45Z failed for B19\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- "
\n",
- "
100%\n",
- "
306/306\n",
- "
[00:01<00:00, 0.00s/it] "
- ],
- "text/plain": [
- "\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 306/306 [00:01<00:00, 0.00s/it]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- ":54: UserWarning: 2019-08-26T13:00Z-2019-08-26T13:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-28T20:15Z-2019-08-28T20:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-29T08:45Z-2019-08-29T08:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-29T16:00Z-2019-08-29T16:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-29T16:30Z-2019-08-29T16:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-29T21:30Z-2019-08-29T21:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-31T09:15Z-2019-08-31T09:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-31T13:15Z-2019-08-31T13:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-31T17:30Z-2019-08-31T17:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-31T18:00Z-2019-08-31T18:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-31T20:45Z-2019-08-31T20:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-31T23:45Z-2019-08-31T23:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-01T01:30Z-2019-09-01T01:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-01T07:45Z-2019-09-01T07:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-01T09:30Z-2019-09-01T09:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-02T04:30Z-2019-09-02T04:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-02T05:00Z-2019-09-02T05:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-02T05:45Z-2019-09-02T05:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-03T04:45Z-2019-09-03T04:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-03T05:30Z-2019-09-03T05:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-04T06:00Z-2019-09-04T06:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-05T18:15Z-2019-09-05T18:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-07T06:00Z-2019-09-07T06:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-08T00:00Z-2019-09-08T00:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-09T07:15Z-2019-09-09T07:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-11T19:45Z-2019-09-11T19:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-12T21:45Z-2019-09-12T21:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-13T04:15Z-2019-09-13T04:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-13T22:45Z-2019-09-13T22:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-14T08:00Z-2019-09-14T08:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-14T17:00Z-2019-09-14T17:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-14T17:30Z-2019-09-14T17:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-14T18:00Z-2019-09-14T18:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-16T13:15Z-2019-09-16T13:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-16T16:30Z-2019-09-16T16:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-16T17:45Z-2019-09-16T17:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-17T01:00Z-2019-09-17T01:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-17T17:00Z-2019-09-17T17:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-17T20:45Z-2019-09-17T20:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-17T23:15Z-2019-09-17T23:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-09-18T16:00Z-2019-09-18T16:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-30T19:45Z-2019-08-30T19:45Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-30T20:30Z-2019-08-30T20:30Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-09T09:15Z-2019-08-09T09:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-09T09:45Z-2019-08-09T09:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-08-09T10:15Z-2019-08-09T10:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- "
\n",
- "
100%\n",
- "
91/91\n",
- "
[00:00<00:00, 0.00s/it] "
- ],
- "text/plain": [
- "\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 91/91 [00:00<00:00, 0.00s/it]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- ":54: UserWarning: 2019-11-28T15:30Z-2019-11-28T15:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-11-04T12:00Z-2019-11-04T12:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-11-16T07:45Z-2019-11-16T07:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-11-16T09:15Z-2019-11-16T09:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-11-16T10:15Z-2019-11-16T10:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-12-05T08:00Z-2019-12-05T08:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-12-31T03:15Z-2019-12-31T03:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-12-31T11:00Z-2019-12-31T11:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2019-12-28T17:30Z-2019-12-28T17:30Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- "
\n",
- "
100%\n",
- "
149/149\n",
- "
[00:00<00:00, 0.00s/it] "
- ],
- "text/plain": [
- "\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 149/149 [00:00<00:00, 0.00s/it]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- ":54: UserWarning: 2020-01-04T19:15Z-2020-01-04T19:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-04T21:45Z-2020-01-04T21:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-05T21:30Z-2020-01-05T21:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-06T00:30Z-2020-01-06T00:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-07T01:15Z-2020-01-07T01:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-11T01:30Z-2020-01-11T01:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-11T05:00Z-2020-01-11T05:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-12T06:00Z-2020-01-12T06:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-12T08:45Z-2020-01-12T08:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-14T10:15Z-2020-01-14T10:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-15T07:45Z-2020-01-15T07:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-15T15:00Z-2020-01-15T15:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-16T14:00Z-2020-01-16T14:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-16T23:45Z-2020-01-16T23:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-17T10:30Z-2020-01-17T10:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-17T11:00Z-2020-01-17T11:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-22T05:45Z-2020-01-22T05:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-27T22:30Z-2020-01-27T22:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-27T23:45Z-2020-01-27T23:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-29T21:30Z-2020-01-29T21:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-29T22:30Z-2020-01-29T22:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-30T11:00Z-2020-01-30T11:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-06T02:45Z-2020-02-06T02:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-10T05:30Z-2020-02-10T05:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-12T07:30Z-2020-02-12T07:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-13T16:15Z-2020-02-13T16:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-16T21:30Z-2020-02-16T21:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-16T23:00Z-2020-02-16T23:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-17T00:00Z-2020-02-17T00:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-17T01:00Z-2020-02-17T01:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-17T01:30Z-2020-02-17T01:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-17T04:00Z-2020-02-17T04:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-17T14:00Z-2020-02-17T14:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-17T18:45Z-2020-02-17T18:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-18T16:45Z-2020-02-18T16:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-02-26T14:00Z-2020-02-26T14:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-03-31T05:15Z-2020-03-31T05:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-01-31T20:45Z-2020-01-31T20:45Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-03-18T11:30Z-2020-03-18T11:30Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- "
\n",
- "
100%\n",
- "
125/125\n",
- "
[00:00<00:00, 0.00s/it] "
- ],
- "text/plain": [
- "\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 125/125 [00:00<00:00, 0.00s/it]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- ":54: UserWarning: 2020-04-09T18:00Z-2020-04-09T18:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-04-14T09:00Z-2020-04-14T09:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-19T11:45Z-2020-06-19T11:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-19T12:15Z-2020-06-19T12:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-26T04:00Z-2020-06-26T04:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-26T05:15Z-2020-06-26T05:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-26T07:45Z-2020-06-26T07:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-26T11:15Z-2020-06-26T11:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-27T02:15Z-2020-06-27T02:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-27T07:45Z-2020-06-27T07:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-28T12:00Z-2020-06-28T12:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T04:45Z-2020-06-29T04:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T05:15Z-2020-06-29T05:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T05:45Z-2020-06-29T05:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T07:15Z-2020-06-29T07:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T07:45Z-2020-06-29T07:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T10:15Z-2020-06-29T10:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T10:45Z-2020-06-29T10:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T13:30Z-2020-06-29T13:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T16:30Z-2020-06-29T16:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T20:00Z-2020-06-29T20:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-29T21:30Z-2020-06-29T21:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T00:45Z-2020-06-30T00:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T01:15Z-2020-06-30T01:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T04:30Z-2020-06-30T04:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T09:00Z-2020-06-30T09:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T09:30Z-2020-06-30T09:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T11:00Z-2020-06-30T11:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T11:30Z-2020-06-30T11:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T12:45Z-2020-06-30T12:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T13:15Z-2020-06-30T13:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T16:15Z-2020-06-30T16:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-06-30T21:30Z-2020-06-30T21:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "\n",
- "
\n",
+ "
\n",
"
100%\n",
- "
148/225\n",
- "
[00:01<00:00, 0.00s/it] "
+ "15/15\n",
+ "[05:37<00:18, 22.45s/it]"
],
"text/plain": [
"\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 225/225 [00:01<00:00, 0.00s/it]"
+ " [████████████████████████████████████████████████████████████] 15/15 [05:37<00:18, 22.45s/it]"
]
},
"metadata": {},
"output_type": "display_data"
},
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- ":54: UserWarning: 2020-07-02T12:00Z-2020-07-02T12:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-28T13:30Z-2020-07-28T13:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-28T14:15Z-2020-07-28T14:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-28T15:00Z-2020-07-28T15:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-02T09:15Z-2020-08-02T09:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-02T11:15Z-2020-08-02T11:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-02T11:45Z-2020-08-02T11:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-02T12:15Z-2020-08-02T12:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-04T20:45Z-2020-08-04T20:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-05T00:30Z-2020-08-05T00:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-05T02:15Z-2020-08-05T02:15Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-05T04:45Z-2020-08-05T04:45Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-05T06:30Z-2020-08-05T06:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-05T11:00Z-2020-08-05T11:00Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-08-05T12:30Z-2020-08-05T12:30Z failed for B04\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-16T07:45Z-2020-07-16T07:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-18T09:00Z-2020-07-18T09:00Z failed for B12\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-01T00:00Z-2020-07-01T00:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-01T16:00Z-2020-07-01T16:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-01T17:30Z-2020-07-01T17:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-02T09:30Z-2020-07-02T09:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-02T10:45Z-2020-07-02T10:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-02T12:15Z-2020-07-02T12:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-02T13:45Z-2020-07-02T13:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-02T15:15Z-2020-07-02T15:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-02T23:15Z-2020-07-02T23:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-03T02:15Z-2020-07-03T02:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-03T02:45Z-2020-07-03T02:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-03T03:30Z-2020-07-03T03:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-03T10:15Z-2020-07-03T10:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-03T12:15Z-2020-07-03T12:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-03T14:30Z-2020-07-03T14:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-04T02:45Z-2020-07-04T02:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-04T06:45Z-2020-07-04T06:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-04T07:15Z-2020-07-04T07:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-04T07:45Z-2020-07-04T07:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-04T08:30Z-2020-07-04T08:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-04T09:15Z-2020-07-04T09:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-04T12:15Z-2020-07-04T12:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-04T12:45Z-2020-07-04T12:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-04T13:15Z-2020-07-04T13:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-05T08:15Z-2020-07-05T08:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-06T06:30Z-2020-07-06T06:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-06T11:00Z-2020-07-06T11:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-06T14:45Z-2020-07-06T14:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-06T19:00Z-2020-07-06T19:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-06T20:30Z-2020-07-06T20:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-06T21:00Z-2020-07-06T21:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-07T01:30Z-2020-07-07T01:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-07T07:00Z-2020-07-07T07:00Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-07T09:30Z-2020-07-07T09:30Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-07T10:45Z-2020-07-07T10:45Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-07-07T12:15Z-2020-07-07T12:15Z failed for B14\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n"
- ]
- },
{
"data": {
"text/html": [
"\n",
- "
\n",
+ "
\n",
"
100%\n",
- "
721/721\n",
- "
[00:01<00:00, 0.00s/it] "
+ "9/9\n",
+ "[02:37<00:14, 17.45s/it]"
],
"text/plain": [
"\u001b[A\u001b[2K\r",
- " [████████████████████████████████████████████████████████████] 721/721 [00:01<00:00, 0.00s/it]"
+ " [████████████████████████████████████████████████████████████] 9/9 [02:37<00:14, 17.45s/it]"
]
},
"metadata": {},
"output_type": "display_data"
},
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- ":54: UserWarning: 2020-10-05T18:00Z-2020-10-05T18:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-06T03:00Z-2020-10-06T03:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-06T12:45Z-2020-10-06T12:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-06T20:45Z-2020-10-06T20:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-09T12:00Z-2020-10-09T12:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-09T12:30Z-2020-10-09T12:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-10T03:45Z-2020-10-10T03:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-10T05:30Z-2020-10-10T05:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T01:30Z-2020-10-11T01:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T02:00Z-2020-10-11T02:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T03:30Z-2020-10-11T03:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T05:15Z-2020-10-11T05:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T05:45Z-2020-10-11T05:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T11:15Z-2020-10-11T11:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T13:30Z-2020-10-11T13:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T14:30Z-2020-10-11T14:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T15:30Z-2020-10-11T15:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T16:00Z-2020-10-11T16:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T16:45Z-2020-10-11T16:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T20:15Z-2020-10-11T20:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-11T22:00Z-2020-10-11T22:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-12T09:00Z-2020-10-12T09:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-12T13:30Z-2020-10-12T13:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-12T23:00Z-2020-10-12T23:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T01:00Z-2020-10-13T01:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T01:45Z-2020-10-13T01:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T04:15Z-2020-10-13T04:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T05:45Z-2020-10-13T05:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T06:15Z-2020-10-13T06:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T11:45Z-2020-10-13T11:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T15:30Z-2020-10-13T15:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T20:00Z-2020-10-13T20:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T20:30Z-2020-10-13T20:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-13T22:30Z-2020-10-13T22:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-14T16:15Z-2020-10-14T16:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-14T19:15Z-2020-10-14T19:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-14T21:15Z-2020-10-14T21:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-15T12:30Z-2020-10-15T12:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-15T16:45Z-2020-10-15T16:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-15T17:30Z-2020-10-15T17:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-15T18:15Z-2020-10-15T18:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-15T19:00Z-2020-10-15T19:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-16T05:00Z-2020-10-16T05:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-16T14:45Z-2020-10-16T14:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-16T16:15Z-2020-10-16T16:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-16T16:45Z-2020-10-16T16:45Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-16T19:00Z-2020-10-16T19:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-17T14:30Z-2020-10-17T14:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-17T16:15Z-2020-10-17T16:15Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-17T18:00Z-2020-10-17T18:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-17T20:00Z-2020-10-17T20:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-18T14:00Z-2020-10-18T14:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-18T14:30Z-2020-10-18T14:30Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- ":54: UserWarning: 2020-10-18T15:00Z-2020-10-18T15:00Z failed for B06\n",
- " warn(f\"{timeseries['Period']['timeInterval']['start']}-{timeseries['Period']['timeInterval']['start']} failed for {psr_type}\")\n",
- "