Econometric_methods.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Econometric models\n",
    "This is a collection of classic econometric models that were used for the kaggle competition.\n",
    "For more info, read README and our blog post."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import os, os.path\n",
    "import itertools\n",
    "import pickle\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pyramid.arima import auto_arima\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import statsmodels.formula.api as smf\n",
    "import statsmodels.tsa.api as smt\n",
    "import statsmodels.api as sm\n",
    "from scipy.stats import describe\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_columns = 12\n",
    "pd.options.display.max_rows = 24\n",
    "\n",
    "# disable warnings in Anaconda\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "# plots inisde jupyter notebook\n",
    "%matplotlib inline\n",
    "\n",
    "sns.set(style='darkgrid', palette='muted')\n",
    "color_scheme = {\n",
    "    'red': '#F1637A',\n",
    "    'green': '#6ABB3E',\n",
    "    'blue': '#3D8DEA',\n",
    "    'black': '#000000'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Seasonality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "additive = {\"op\": lambda a,b: a+b, \"inv\": lambda a,b: a-b}\n",
    "multiplicative = {\"op\": lambda a,b: a*b, \"inv\": lambda a,b: a/b}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Choose seasonality model from above:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "seasonality_model = additive\n",
    "#seasonality_model = multiplicative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorize_by_week_of_year = lambda df: df.index.dayofyear // 7\n",
    "categorize_by_day_of_week = lambda df: df.index.dayofweek"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_seasonality(series, categorization):\n",
    "    \"\"\"\n",
    "    Computes seasonal component parameters based on provided series.\n",
    "    \n",
    "    :type series: pd.Series\n",
    "    :param categorization: Function used to split values into various periods of the season.\n",
    "    :type categorization: pd.DataFrame -> some categorical type, eg. int\n",
    "    \"\"\"\n",
    "    df = pd.DataFrame()\n",
    "    df[\"values\"] = series\n",
    "    df.index = series.index\n",
    "    df[\"cat\"] = categorization(df)\n",
    "    return df.groupby(by=\"cat\")[\"values\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def alter_series_by_season(series, categorization, seasonality, op):\n",
    "    df = pd.DataFrame()\n",
    "    df[\"values\"] = series\n",
    "    df[\"values\"] = series\n",
    "    df[\"cat\"] = categorization(df)\n",
    "    return op(df[\"values\"], df[\"cat\"].map(seasonality))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_seasonal_component(series, categorization, seasonality):\n",
    "    \"\"\"\n",
    "    Add previously computed seasonal component back into a deseasonalized series.\n",
    "    \n",
    "    :type series: pd.Series\n",
    "    :param categorization: Function used to split values into various periods of the season.\n",
    "    :type categorization: pd.DataFrame -> some categorical type, eg. int\n",
    "    :param seasonality: value returned from compute_seasonality\n",
    "    :returns: Series with added seasonal component.\n",
    "    :rtype: pd.Series\n",
    "    \"\"\"\n",
    "    return alter_series_by_season(series, categorization, seasonality, seasonality_model[\"op\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_seasonal_component(series, categorization, seasonality):\n",
    "    \"\"\"\n",
    "    Try removing a previously computed seasonal component.\n",
    "    \n",
    "    :type series: pd.Series\n",
    "    :param categorization: Function used to split values into various periods of the season.\n",
    "    :type categorization: pd.DataFrame -> some categorical type, eg. int\n",
    "    :param seasonality: value returned from compute_seasonality\n",
    "    :returns: Deseasonalized series.\n",
    "    :rtype: pd.Series\n",
    "    \"\"\"\n",
    "    return alter_series_by_season(series, categorization, seasonality, seasonality_model[\"inv\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Holt-Winters and SARIMA Models Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_forecast(data, categorization, trainer, forecaster, deseasonize, steps_to_forecast=90):\n",
    "    \"\"\"\n",
    "    Split input data, deseasonalizes train data,\n",
    "    train using trainer (data -> model),\n",
    "    forecast using forecaster\n",
    "    \n",
    "    predicts values and applies seasonalization and returns predicted vs actual values\n",
    "    \n",
    "    :param data: dataset with the training data\n",
    "    :param categorization: Function used to split values into various periods of the season.\n",
    "    :type categorization: pd.DataFrame -> some categorical type, eg. int\n",
    "    :param trainer: Function used to train the model\n",
    "    :type trainer: pd.DataFrame -> model\n",
    "    :param forecaster: (model, steps) -> prediction\n",
    "    :param steps_to_forecast:  number of steps to forecast\n",
    "    :returns:  a dataframe with:\n",
    "                                date\n",
    "                                sales - true values\n",
    "                                forecast - forecasted values\n",
    "    \"\"\"\n",
    "    \n",
    "    #prepare trainig and validation datasets\n",
    "    df_train = data.iloc[:-365].copy()\n",
    "    df_validation = data.iloc[-365:].copy()\n",
    "    df_validation.index = pd.DatetimeIndex(df_validation[\"date\"])\n",
    "    df_train.index = pd.DatetimeIndex(df_train[\"date\"])\n",
    "    \n",
    "    if deseasonize:\n",
    "        seas = compute_seasonality(df_train[\"sales\"], categorization)\n",
    "        series = remove_seasonal_component(df_train[\"sales\"], categorization, seas)\n",
    "        df_train[\"sales\"] = series\n",
    "    \n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "\n",
    "    # train\n",
    "    model = trainer(df_train)\n",
    "    \n",
    "    # forecast\n",
    "    forecast = forecaster(model, steps_to_forecast)\n",
    "    \n",
    "    # Create the pandas series from the forecast\n",
    "    forecast = pd.Series(forecast)\n",
    "    forecast.name = \"sales\"\n",
    "    forecast.index = pd.DatetimeIndex(start='2017-01-01', \n",
    "                                      freq=\"D\",\n",
    "                                      periods = forecast.size)\n",
    "    \n",
    "    if deseasonize:\n",
    "        forecast = add_seasonal_component(forecast, categorization, seas)\n",
    "    \n",
    "    final_forecast = pd.DataFrame()\n",
    "    final_forecast['real_values'] = df_validation[\"sales\"][:steps_to_forecast]\n",
    "    final_forecast['forecast'] = forecast\n",
    "    \n",
    "    return final_forecast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_single_df(data, store, item):\n",
    "    \"\"\"\n",
    "    Extract single store/item time series from provided \n",
    "    dataset\n",
    "    \n",
    "    :param data: Pandas dataframe with multiple timeseries\n",
    "    :param store: number of the store\n",
    "    :param item: number of the item\n",
    "    :returns: Pandas dataframe with single store/item time series\n",
    "    \"\"\"\n",
    "    \n",
    "    df_single = data.loc[(data.store == store) & (data.item == item),[\"date\", \"sales\"]].copy()\n",
    "    df_single.reset_index(drop=True, inplace=True)\n",
    "    df_single.date = pd.to_datetime(df_single.date)\n",
    "    return df_single"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_all_models(data, ids, categorization, trainer, forecaster, deseasonize, steps_to_forecast=90):\n",
    "    \"\"\"\n",
    "    Train the models and use them to make forecast for all of the individual\n",
    "    time series separately\n",
    "    \n",
    "    :params data: dataframe with multiple time series\n",
    "    :params ids: list of tuples with stores and items\n",
    "    :param categorization: Function used to split values into various periods of the season.\n",
    "    :type categorization: pd.DataFrame -> some categorical type, eg. int\n",
    "    :param trainer: Function used to train the model\n",
    "    :type trainer: pd.DataFrame -> model\n",
    "    :param forecaster: (model, steps) -> prediction\n",
    "    :param steps_to_forecast: number of steps to forecast\n",
    "    \"\"\"\n",
    "    \n",
    "    all_models_forecast = {}\n",
    "    all_models_smape = np.array([])\n",
    "    number_of_time_series = 0\n",
    "    for store, item in ids:\n",
    "        single_time_series = extract_single_df(data, store, item)\n",
    "        predictions = train_and_forecast(single_time_series, categorization,\n",
    "                                         trainer, forecaster, deseasonize, steps_to_forecast)\n",
    "        score = smape(predictions['real_values'], predictions['forecast'])\n",
    "        results = {\n",
    "            \"item\": item,\n",
    "            \"store\": store,\n",
    "            \"predictions\": predictions,\n",
    "            \"smape\": score\n",
    "        }\n",
    "        print(score)\n",
    "        all_models_smape = np.append(all_models_smape, score)\n",
    "        all_models_forecast[str(store) + str(item)] = results\n",
    "        number_of_time_series += 1\n",
    "    forecast_smape = np.sum(all_models_smape) / number_of_time_series\n",
    "        \n",
    "    return results, forecast_smape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def smape(y, y_pred):\n",
    "    \"\"\"\n",
    "    compute the SMAPE metrics\n",
    "    \n",
    "    :param y: array with true values\n",
    "    :param y: array with forcasted values\n",
    "    :returns: average smape metrics for the given period\n",
    "    \"\"\"\n",
    "    \n",
    "    div = (abs(y_pred) + abs(y)) / 2\n",
    "    errors = abs(y_pred - y) / div\n",
    "        \n",
    "    smape = np.sum(errors) / len(y)\n",
    "    return smape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_avg_smape (df_y, df_y_pred):\n",
    "    \"\"\"\n",
    "    Compute average SMAPE of multiple forecast\n",
    "    \n",
    "    :param df_y: data series with real values\n",
    "    :param df_y_pred: dataframe with multiple forecasts\n",
    "    :returns: average SMAPE of all forecasts\n",
    "    \"\"\"\n",
    "    \n",
    "    avg_smape = 0\n",
    "    for i in range(df_y_pred.shape[1]):\n",
    "        err = smape(y=df_y.iloc[:,i],\n",
    "                              y_pred=df_y_pred.iloc[:,i])\n",
    "        avg_smape += err\n",
    "\n",
    "    avg_smape /= df_y_pred.shape[1]\n",
    "    return avg_smape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data import & preparation \n",
    "In order for this to work, download all datasets from kaggle competition:  \n",
    "https://www.kaggle.com/c/demand-forecasting-kernels-only  \n",
    "and place them in `../data/` folder.  \n",
    "We could not append datasets to our repo because of copyrights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import data\n",
    "dir_name = \"../data/\" #insert your own path here\n",
    "file_name = \"train.csv\"\n",
    "\n",
    "filepath_train = os.path.abspath(os.path.join(os.getcwd(), dir_name, file_name ))\n",
    "\n",
    "store_item_data = pd.read_csv(filepath_train)\n",
    "store_item_data.index = pd.DatetimeIndex(store_item_data['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_stores = 10\n",
    "number_of_items = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = list(itertools.product(range(1,number_of_stores+1), range(1,number_of_items+1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Holt-Winters Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a trainer which can be use to train all of the models\n",
    "hw_trainer = lambda df: smt.ExponentialSmoothing(endog=df.loc[-365:,'sales'], damped=False,\n",
    "                         trend='add',seasonal='add', seasonal_periods=7).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a forecaster function to get the forcasts from all of the models\n",
    "hw_forecaster = lambda model, steps: model.predict(steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "hw_results, hw_smape = compute_all_models(store_item_data, ids, categorize_by_week_of_year,\n",
    "                                         hw_trainer, hw_forecaster, False)\n",
    "print(\"Holt Winters method SMAPE on the validation set: \", hw_smape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SARIMA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a trainer which can be use to grid search and train all of the models\n",
    "arima_trainer = lambda df: auto_arima(df.loc[-365:,'sales'], m=7, n_jobs=1, max_p=7, max_q=7, max_P=7,\n",
    "                          max_Q=7, max_order=12, trend='c', max_iter=100,\n",
    "                          trace=False, error_action='ignore', suppress_warnings=True,\n",
    "                          stepwise=True, random=False, n_fits=40, random_state=44443)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a forecaster function to get the forcasts from all of the models\n",
    "arima_forecaster = lambda model, steps: model.predict(steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sarima_results, sarima_smape = compute_all_models(store_item_data, ids, categorize_by_week_of_year,\n",
    "                                         arima_trainer, arima_forecaster, True)\n",
    "print(\"SARIMA model SMAPE on the validation set: \", sarima_smape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VAR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def var_train_and_forecast(data_train, data_validation, categorization, max_lags=16,ic='aic',\n",
    "                            plots=False):\n",
    "    \"\"\"\n",
    "    Apply deseasonalization and differencing to the time series and\n",
    "    create VAR model with optimal number of lags, make forecast and compute smape\n",
    "    \n",
    "    :param data_train: training multivariete time series\n",
    "    :param data_validation: validation  multivariete time series\n",
    "    :param max_lags: maximum number of lags in VAR model\n",
    "    :type max_lags: int\n",
    "    :param ic: information criterion used to choose best VAR model\n",
    "    :param plots: if True  forecast vs real_values plot will be printed\n",
    "    \n",
    "    returns: smape metric\n",
    "    \"\"\"\n",
    "    \n",
    "    #DESEASONALIZATION\n",
    "    data_train_deseasonalized = pd.DataFrame()\n",
    "    seasonalized_all = pd.DataFrame()\n",
    "\n",
    "    for col in  data_train.columns.values:\n",
    "        seasonalized_all[col] = compute_seasonality(data_train[col], categorization)\n",
    "        series = remove_seasonal_component(data_train[col], categorization, seasonalized_all[col])\n",
    "        data_train_deseasonalized[col] = series\n",
    "    data_train_deseasonalized.index = pd.DatetimeIndex(start=\"01-01-2013\",\n",
    "                                                       periods=1461, freq='D')\n",
    "    \n",
    "    #TIME SERIES DIFFERENCING\n",
    "    data_train_deseasonalized_differenced = data_train_deseasonalized.diff().iloc[1:]\n",
    "    data_train_deseasonalized_differenced = data_train_deseasonalized_differenced.asfreq('D')\n",
    "    \n",
    "    #MODELING\n",
    "    var_model = smt.VAR(data_train_deseasonalized_differenced,\n",
    "                        dates=data_train_deseasonalized_differenced.index)\n",
    "    \n",
    "    var_model_results = var_model.fit(maxlags=max_lags, ic=ic, verbose=True)\n",
    "    lags = var_model_results.k_ar\n",
    "    \n",
    "    #FORECASTING\n",
    "    forecast = pd.DataFrame(var_model_results.forecast(\n",
    "                            data_train_deseasonalized_differenced.values[-lags:], 90))\n",
    "    forecast.columns +=1\n",
    "    \n",
    "    #FORCAST REVERSE DIFFERENCING\n",
    "    var_forecast_diff = data_train_deseasonalized.tail(1)\n",
    "\n",
    "    var_forecast_diff = var_forecast_diff.append(forecast)\n",
    "    var_forecast = var_forecast_diff.cumsum().iloc[1:]\n",
    "    var_forecast.index = pd.DatetimeIndex(start='01-01-2017', periods=90, freq='D')\n",
    "    \n",
    "\n",
    "    \n",
    "    # FORCAST - APPLY SEASONALITY\n",
    "    corrected_forecast = pd.DataFrame()\n",
    "    for col in var_forecast.columns.values:\n",
    "        tmp_forecast = pd.DataFrame()\n",
    "        tmp_forecast = add_seasonal_component(var_forecast[col].copy(),\n",
    "                                        categorize_by_week_of_year,\n",
    "                                        seasonalized_all[col])\n",
    "        corrected_forecast[col] = tmp_forecast\n",
    "    corrected_forecast.index = pd.DatetimeIndex(start='01-01-2017', periods=90,freq='D')\n",
    "    \n",
    "    # forecast vs real_values plot\n",
    "    if plots:\n",
    "        plt.figure()\n",
    "        corrected_forecast.iloc[:,0].plot(figsize=(12,16), color='b')\n",
    "        data_validation.iloc[:90,0].plot(color='r')\n",
    "        plt.show()\n",
    "    smape = compute_avg_smape(df_y=data_validation.iloc[:90], df_y_pred=corrected_forecast)\n",
    "    print(smape)\n",
    "    return smape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_multiple_var (data, num_time_series):\n",
    "    \"\"\"\n",
    "    Compute and evaluate VAR model for every batch of multivariete time series\n",
    "    :param data: multiindexed columns dataframe with multivariete time series\n",
    "    :param number_of_series: number of mutivariete time series\n",
    "    :returns: average SMAPE metric\n",
    "    \"\"\"\n",
    "    smape_all = []\n",
    "    for i in range(1,num_time_series+1):\n",
    "        single_time_series = data.loc[:,i]\n",
    "    \n",
    "        single_time_series_train = single_time_series.iloc[:-365]\n",
    "        single_time_series_validation = single_time_series.iloc[-365:]\n",
    "    \n",
    "        smape = var_train_and_forecast(single_time_series_train, single_time_series_validation, categorize_by_week_of_year,\n",
    "                                    plots=False)\n",
    "        smape_all.append(smape)\n",
    "        \n",
    "    return smape_all"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a model for each store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>store</th>\n",
       "      <th colspan=\"6\" halign=\"left\">1</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"6\" halign=\"left\">10</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>...</th>\n",
       "      <th>45</th>\n",
       "      <th>46</th>\n",
       "      <th>47</th>\n",
       "      <th>48</th>\n",
       "      <th>49</th>\n",
       "      <th>50</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2013-01-01</th>\n",
       "      <td>13</td>\n",
       "      <td>33</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>31</td>\n",
       "      <td>...</td>\n",
       "      <td>45</td>\n",
       "      <td>37</td>\n",
       "      <td>11</td>\n",
       "      <td>25</td>\n",
       "      <td>17</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-02</th>\n",
       "      <td>11</td>\n",
       "      <td>43</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>45</td>\n",
       "      <td>33</td>\n",
       "      <td>13</td>\n",
       "      <td>24</td>\n",
       "      <td>13</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-03</th>\n",
       "      <td>14</td>\n",
       "      <td>23</td>\n",
       "      <td>14</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>56</td>\n",
       "      <td>28</td>\n",
       "      <td>16</td>\n",
       "      <td>29</td>\n",
       "      <td>19</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-04</th>\n",
       "      <td>13</td>\n",
       "      <td>18</td>\n",
       "      <td>10</td>\n",
       "      <td>19</td>\n",
       "      <td>9</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>50</td>\n",
       "      <td>44</td>\n",
       "      <td>11</td>\n",
       "      <td>39</td>\n",
       "      <td>23</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-05</th>\n",
       "      <td>10</td>\n",
       "      <td>34</td>\n",
       "      <td>23</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>31</td>\n",
       "      <td>...</td>\n",
       "      <td>62</td>\n",
       "      <td>45</td>\n",
       "      <td>16</td>\n",
       "      <td>34</td>\n",
       "      <td>22</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 500 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "store       1                      ...  10                    \n",
       "item        1   2   3   4   5   6  ...  45  46  47  48  49  50\n",
       "date                               ...                        \n",
       "2013-01-01  13  33  15  10  11  31 ...  45  37  11  25  17  33\n",
       "2013-01-02  11  43  30  11   6  36 ...  45  33  13  24  13  37\n",
       "2013-01-03  14  23  14   8   8  18 ...  56  28  16  29  19  46\n",
       "2013-01-04  13  18  10  19   9  19 ...  50  44  11  39  23  51\n",
       "2013-01-05  10  34  23  12   8  31 ...  62  45  16  34  22  41\n",
       "\n",
       "[5 rows x 500 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stores_time_series = pd.pivot_table(\n",
    "    columns=['store','item'], values='sales',\n",
    "    index=store_item_data.index, data=store_item_data).asfreq('D')\n",
    "stores_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.1614194423273425\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.13758328818926077\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.1526445758664326\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.15339676809920877\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.17406155815250476\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.17554093888197206\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.18497158371207476\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.1408921398129204\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.15685467489594204\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 1, FPE -> 6, HQIC -> 2>\n",
      "Using 16 based on aic criterion\n",
      "0.15031806865439024\n"
     ]
    }
   ],
   "source": [
    "all_stores_smape = evaluate_multiple_var(stores_time_series, number_of_stores)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD4CAYAAAANbUbJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl0lHWeLvDnfWtLbdkrCQQCBMKSsEuAGAgSNMomuCD2YiOjF+f2Yvfh2K22t/vcYbrHsee2p3uc6e7hesfREe1WVHAXQbYg+xYCIexLQsheldrX9/5RkNYWSFLbW8vzOcejVqjk+0uqnrz8lu8rSJIkgYiIkpoodwFERBR9DHsiohTAsCciSgEMeyKiFMCwJyJKAcpYfrH2dmssv1zUZGXp0N3tkLuMqOIYkwPHmBxMJmPYn4NX9iFQKhVylxB1HGNy4BjpOoY9EVEKYNgTEaUAhj0RUQpg2BMRpQCGPRFRCmDYExGlAIY9EVEKYNgTEaWAPk/QtrS04Gc/+xk6OjogiiIeeughrFixAi+99BLeeustZGdnAwBWr16NOXPmRL1gIiIauD7DXqFQ4JlnnkFZWRlsNhseeOABVFZWAgAeffRRPPbYY1EvkqJja13PTT9mMLphs7pjWE3/zJ2YLncJRAmpz7DPy8tDXl4eAMBgMKC4uBitra1RL4yIiCJnQI3Qmpqa0NDQgEmTJuHQoUNYt24dNmzYgPHjx+OZZ55BRkbGLZ+flaVLmj4WkWhMJDeD8dZX7gajJkaV9F+kv+/J8HPsC8dIACD09x60drsdjzzyCP7+7/8eNTU16OjoQFZWFgRBwO9//3u0tbXh+eefv+XnSJaulyaTMSnGcutpHE3ST+Mky8/xVjjG5BCzrpderxdPPvkkFi9ejJqaGgBAbm4uFAoFRFHEsmXLcOzYsbCLISKi6Ogz7CVJwnPPPYfi4mKsXLmy9/G2trbe/968eTNKSkqiUyEREYWtzzn7gwcPYuPGjRg9ejSWLFkCILjN8sMPP8TJkycBAIWFhVizZk10KyUiopD1GfbTpk1DY2PjNx7nnnoiosTBE7RERCmAYU9ElAIY9kREKYBhT0SUAhj2REQpgGFPRJQCGPZERCmAYU9ElAIY9kREKYBhT0SUAhj2REQpgGFPRJQCGPZERCmAYU9ElAIY9kREKYBhTwmjn7dLJqIb6PPmJURykCQJXVYfzl51o9Xshd0VgNMTwDtfdiE3XYXh+RpUjDVgdGEaREGQu1yiuMewp7hzpcuD/ads6LL5AQCiAOjTRORlKCEIApo6PDjf6sbWuh7kpivxwO3ZqBhnYOgT3QLDnuKG0xPA3kYbzre6AQDD8tQYWZCGIblqKMRgkM+dmI5AQEJDkxNfNtiw56QN//FpGzYfseDRO00YlqeRcwhEcYthT3Ghy+rD5iMW2N0BmNKVmDnWgNx01Q3/rCgKKCvSoaxIh/sqsvCXnV3Y22jDmjebsWJeLqrGp8e4eqL4xwVakt2ldjc+OtANuzuAqSN1WFieedOg/1u56Sr8YGE+Vi8tgEop4OVN7fivze3wB7iYS/RVDHuS1eV2N76o6wGk4BTNpBF6CCHMvU8u1mPNd4agyKTGF3U9+OPHrfD5GfhE1zHsSTYtXR5sPdYDhQDUTM3E8DDn2/MyVXhueSHGFKZh3yk7/u3Dq/D6GPhEAMOeZNJp9WHzUQskCaielIH8zP5N2/RFqxbx1P2DUDpUi0NnHVj7WRsC3J9PxLCn2HN7A/iizgKfH5gzPh2FOeqIfn6NSsTqpQUYXZiGvY02vLWzK6KfnygRMewppiRJwvb6HticAUwaocPw/OhslVSrRPzk3gIMylbh4wNmfH7YEpWvQ5QoGPYUU0fPO9Dc6UVhjgqTi3VR/VoGrQJP3TcIGToF1m3rQMNlZ1S/HlE8Y9hTzHT0eHHkvAN6jYg549NjcuLVlKHCDxfnQxCAf/+oFV1WX9S/JlE8YthTTPj8EnYet0KSgFllRmhUsXvpjSnU4ltVOehx+PHSh1e5JZNSEsOeYuLwOTvMdj/GDknD4OzILsj2x11TMnD7WAPOtrixfhcXbCn1MOwp6jp6vDh+0QmjVsS0EoMsNQiCgEfvNCE/M7hgW3/RIUsdRHJh2FNUSZKEPSdtkADcPs4IlUK+zpRpahH/c0EeFCKw9tM2mG2cv6fU0WfYt7S04JFHHsH8+fOxcOFCvPrqqwAAs9mMlStXoqamBitXroTFwq1t9E2nr7jQ3uPDiHyNLNM3f6u4IA0PVmbDbPfjX9+7zBuiUMroM+wVCgWeeeYZfPLJJ/jLX/6CN954A2fOnMHatWtRUVGBTZs2oaKiAmvXro1FvZRA3N4ADpyxQ6kQUF6il7ucXvOnZWJMYRp2n7Bgb6NN7nKIYqLPsM/Ly0NZWRkAwGAwoLi4GK2trdiyZQuWLl0KAFi6dCk2b94c3Uop4Rw+54DbK2FysQ76NIXc5fQSBQGP350HjUrAa190wGLndA4lvwH1s29qakJDQwMmTZqEzs5O5OXlAQj+Qujq6nuHQ1aWDkpl/Lzpw2EyGeUuIWwGo7uPj4d+utVs86GxyYkMvRLTSzN7bz4Srkh9300mYOXdfvzpw2b8udaM574zPKRum4kgGV6rfUmFMYar32Fvt9vx5JNP4uc//zkMhtB2VHR3J8cOCJPJiPZ2q9xlhM1mvXnYG4yaW368LzvrehCQgKnFOjjtnpA/z9+K5Pd9cUUuth7uxK7jFny46ypmjpFnp1A0Jctr9VZSZYzh6tduHK/XiyeffBKLFy9GTU0NACAnJwdtbW0AgLa2NmRnZ4ddDCWHdosXF9rcMKUrMSxP/kXZmxFFAY/X5EGtFPDalnb0ODidQ8mrz7CXJAnPPfcciouLsXLlyt7Hq6ursWHDBgDAhg0bMG/evOhVSQlDkiTsP20HAEwrCe1GJLGUn6XCslnZsLkCeHVLh9zlEEVNn2F/8OBBbNy4EXv27MGSJUuwZMkSbN++HatWrcKuXbtQU1ODXbt2YdWqVbGol+JcS5cXrWYvhuSqUZAVv1f1X3XXlAyMLkzD/tN27DvF3TmUnPqcs582bRoaGxtv+LHre+6JgOBV/eFzwav6KVHuaBlJoiDg8RoTnnutCa990YHSoVoYtMmxkYDoOp6gpYi50uVFm8WHIpO63zcMjxcFWWrcV5GFHocfb+7olLscoohj2FNESJKEw2eDV/WTi+PnANVAzJ+WiWF5auw8bmXvHEo6A9pnT3QzV7q8aO/xYZhJjRxj9F5WW+t6Iva5DEb3N7aXThquw6V2D/7wUSuWVmTL2ssnEm40xv6YOzE9CtWQnHhlTxFx9HzwSnhSgl7VX5eTrsL4Ii1srkDv31SIkgHDnsLWag7uwCnMie5VfaxMLtbDqFXgxCUn2i1eucshigiGPYXt2IXgVf3E4VqZK4kMpUJA5TgDJAC1J6zwB9gZkxIfw57C0mXz4XKHB3kZyoTZV98fg7LVGF2YBrPd3/vLjCiRMewpLPXXr+pHJM6++v6aNkoPrVrE0fMOmNkZkxIcw55C5nD7ca7VjUy9AkNykueq/jqNSkTFWAMCErDrhJU3OqGExrCnkDVcdkGSgLIibdz3wAnVsDwNhuep0Wbx4WSTS+5yiELGsKeQ+PwSGpud0KgEFBekyV1OVM0YY4RaKeDAGTtsLr/c5RCFhGFPITnb4oLbK2FMoRbKBD941BedRkR5iR4+v4TdDTZO51BCYtjTgEmShOOXnRAFYNzQ5L6qv65kcBoGZavQ1OnBudbQb+pCJBeGPQ3YlS4vLHY/RuRroNOkRndIQRBQOdYIhQjsbbTB4Q7IXRLRgDDsacCOXwputywtSo5DVP1l1Clw2yg93F4Juxq4O4cSC8OeBsRs96G504v8TGXCtTGOhNKh2uB0TocHp5q5O4cSB8OeBuTEJScAoLQo+Q5R9YcgCJhdGtyds++UDT0O7s6hxMCwp35zeQI40+KCIU1EkSn5DlH1lz5NgYqxBvgCwI76HgTYO4cSAMOe+u3UFRf8AWDcUC3EJD1E1V/FBWkoLtCgvceHOvbOoQTAsKd+kaTgISqFGNyGSMDMMQboNCKOnHewFTLFPYY99Utzpwc2ZwDFBWnQqPiyAYK9c2aXGSFJwPZ6Kzw+bsek+MV3LfXL9b4wY4fwqv6rBmerMWGYFlanHzuPczsmxS+GPfXJ5vKjqcOD3PTU3G7Zl6kj9SjIUuFSuwf113YrEcUbhj316VSzCxKAMYW8qr8RURRwx/h0aNUiDp6x42q3R+6SiL6BYU+3FAhIONXsglqZ/N0tw6HViJg7MR0AsO2YFQ43999TfGHY0y1dbPfA6Qlg1KC0pO9uGa78TBXKR+nh9ASw7ZiV++8prjDs6ZZONgXnoMdwYbZfSou0GJ6nRqvZi32n7XKXQ9SLYU83Zbb7cLXbi4IsFTL1SrnLSQiCIKCy1IhMvQINl504cYkHrig+MOzpphp7t1umVnfLcKmVIu6anAGtWsTeU3ZcbGP/e5Ifw55uyOsL9sHRqkUMS+E+OKEyaBW4c3I6lCKwvb6HJ2xJdgx7uqEzzU54fBJKBqdBFLkwG4rcdBXmTEhHIABsPmqB1ckdOiQfhj3d0ImLwcXF0dxbH5YikwYzxhjg8kj4/LAFbi9bKpA8+gz7Z599FhUVFVi0aFHvYy+99BJmz56NJUuWYMmSJdi+fXtUi6TYsth9uNLpweBsFYza1LjtYDSNG6pFWZEWFocfmw5b2EOHZNFn2N9///14+eWXv/H4o48+io0bN2Ljxo2YM2dOVIojeZy6ElyYZXfLyCkv0WPUIA06enz4/LAFXh/34FNs9Rn25eXlyMjIiEUtFAcCAQlnWlxIU4koMmnkLidpXN+SOSJfgzaLD5uPWuDzM/ApdkLePL1u3Tps2LAB48ePxzPPPNOvXwhZWToolckxLWAyGeUuIWwG4ze3BJ694oTLI2HSSD0yM5P/yt5gjO0vtPkzNfhsfxfOXnFi+3ErFs3MhSLKJ5NDGWOivb4TrV45hBT23/rWt/D9738fgiDg97//Pf75n/8Zzz//fJ/P6+5OjgMmJpMR7e1WucsIm836zbCvOxscV+kw3Q0/nkwMRo0sY6wcq4fb48elNjc++LIdcyemQxGlHU+hjjGRXt/J8n68lUj8MgtpN05ubi4UCgVEUcSyZctw7NixsAsh+dldfjRfb2Wcwb310aIQBcydkI7B2Spc7vBga10P/OyjQ1EWUti3tbX1/vfmzZtRUlISsYJIPmdagq2MR3NhNuqUCgHzJmX0Bv4WzuFTlPU5jbN69Wrs27cP3d3dqKqqwo9+9CPs27cPJ0+eBAAUFhZizZo1US+UokuSJJy+4oJSBEYUcGE2Fq4H/tZjPWjq8GDzUQvmTcqAit1FKQr6DPsXX3zxG48tW7YsKsWQfFq6vbA6Axg1SAO1kmftYkWpEFA9MR3bjvXgUrsHnx+24K7J6VDxZ0ARxlcUAQBOX9tbP7qQTc9i7foc/vA8DVrNXh68oqhg2BPc3gAutrmRoVMgL4OtjOUgigLmjDeiuCC4D/+zQ2ytQJHFsCecveqGPxA8MSsInC+WiygKmF1m7D1p+9khC1weBj5FBsM+xUmShFPNTggCMGoQd+HITRQEzCo1YnRhGjqtPnx6yAwnA58igGGf4jqtPnTb/Biaq4ZWw5dDPBAEAbePNWDskDR02/z49KAZDjcDn8LDd3eKO9V8fWGWV/XxRBAEzBxjQFmRFma7H58cNMPuYj98Ch3DPoX5/BLOXXVDpxFRmMMTs/FGEASUl+gxYZgWPY5g4NsY+BQihn0Ku9Dqhtd/7W5UXJiNS4Ig4LZRekwaoYPVGcAnB8y84xWFhGGfwti3PjEIgoCpI/WYUqyDzRUM/B4HA58GhmGfolq6PWg1ezGId6NKGJOL9Zg2Sg+7O4CPD5hhsfvkLokSCMM+Re04FmwJy6ZniWXCcB2mj9bD6Qngs0MWLtpSvzHsU5DPL6H2hBVqpcC7USWgsiIdbhsZvMLfxJuYUz8x7FPQ0fMOWBx+jByUBiU7LCakCcO1GDc0uC1zy9EetkemPjHsU9D2+h4AnMJJZIIgYMZoPYbnB5unba/vQYA3QKFbYNinmC6rD0fPO1Ccr0G2kU3PEpkgCKgqM2JQlgqX2j3Y3WiDJDHw6cYY9imm9oQVkgRUTeANmpOBQhRQPSkd2QYlTjW7UH/RKXdJFKcY9ikkIEnYUd8DtVJAxRiGfbJQK0XcNSUDOo2IA2fsaOrwyF0SxSGGfQo5edmFNosP00cb2PQsyeg0IqonpkMhBtdkeOiK/hbf8Snk+sLsHE7hJCVThgoVY43w+CRsOWqBl3e7oq9g2KcIu8uPA6ftGJSl4i6cJFYyOK13S+aO41Yu2FIvhn2K2NVgg9cvoWq8kXejSnLTS/QouLZDZ3+jVe5yKE4w7FOAJEnYfqwHChGYVcopnGQnXruBuSFNxN6GHjR3csGWGPYp4dxVNy53eDB1pB4Zeu6tTwVpahFzJ6ZDFICdx628ly0x7FPBtmPXF2bTZa6EYik3XYWK0gw4PYFr5ys4f5/KGPZJzukJYE+jDbnpSowfppW7HIqxKSUGDM5W4XKHBw1NLrnLIRkx7JPcnpM2uL0SqsqMvBtVChIEAbPLjNCoBBw4bUOXjT3wUxXDPsltr++BIABV4zmFk6p0GgVmlRrhDwDbj7FDZqpi2CexS+1unLvqxsThOjY9S3FFJg3GDUmD2e7H/tM2ucshGTDsk9j1hdk7uDBLAKaVGJCpV+BkkwtXurgdM9Uw7JOUxxvAlw02ZOgVmDRCJ3c5FAeUiuD8vSAAuxqs8Po4nZNKGPZJav9pOxzuAKrKjLwbFfXKTVdh/DAtbM4ADp61y10OxRDDPkldn8Lhwiz9rckj9MjQKdBw2YlWs1fucihG+gz7Z599FhUVFVi0aFHvY2azGStXrkRNTQ1WrlwJi8US1SJpYFq6PGhsdqF0qBb5mSq5y6E4o1QIqLzWNqP2hJW7c1JEn2F///334+WXX/7aY2vXrkVFRQU2bdqEiooKrF27NmoF0sD9dWGWfXDoxvIzVSgdqkWPw48j5zidkwr6DPvy8nJkZGR87bEtW7Zg6dKlAIClS5di8+bN0amOBsznl1B7wgp9mojbRhnkLofi2NRRehi0IuovOtFu4XROsgtp83VnZyfy8vIAAHl5eejq6urX87KydFAqFaF8ybhjMsXnVfPOY2ZYnQEsrTRh8KBbz9cbjO4+Pq6JZGlxKdXHeOdt2dhQ24E9p+xYfkceRDG4mB+vr++bSbR65RDTkzbd3Y5YfrmoMZmMaG+Pzz7h7+9qBQDMGJXWZ402683D3mDU3PLjyYBjBLLSBJQMTsPpKy7sbzCjrCi4TTdeX983Es/vx0iJxC+zkHbj5OTkoK2tDQDQ1taG7OzssAuh8LVbvDh+0YmSwWkozFHLXQ4liGmj9NCoBBw664DdxXvXJquQwr66uhobNmwAAGzYsAHz5s2LaFEUmu31PZAAzBnPv9JS/6WpRdw2Sg+fX8L+01ysTVZ9hv3q1avx8MMP4/z586iqqsLbb7+NVatWYdeuXaipqcGuXbuwatWqWNRKt+DzS9h2zAqdRsSMMVyYpYEZPTgNpgwlzre6eWerJNXnnP2LL754w8dfffXViBdDoTtwxo4ehx93T82ARsWzcjQwgiCgYqwRH+ztxu6TNjw0OxtqJV9HyYQ/zSSx5UjwYFv1JJ6YpdDkGJUYN1QLq9OPjw+Y5S6HIoxhnwSaOtxobHahrEiLQVlcmKXQTRmpg1Yt4oO9ZrZSSDIM+yTwxdHgidl5vKqnMKmVIqaP1sPrl7BuW4fc5VAEMewTnNMTQG2DFVkGBaaM1MtdDiWBEfnBG50cOedA3fnkOBtDDPuEt7vBCpdHwtyJ6VCIbGVM4RMEAd+ZmwtBANZt62CjtCTBsE9gkiRhy9EeKETgDrYypggqMmlQPTEdLd1efH6EXW2TAcM+gZ2+4sLlDg9uG6VHpoH3mKXIuv/2bOg1Ijbs7obF7pO7HAoTwz6BbeldmM3o408SDZxRq8ADldlwegJ4u7Z/zQ4pfjHsE1SPw4d9p2wYnK3C2CFpcpdDSWruxHQMzVVj53Erzl11yV0OhYFhn6C+ONoDfyB4VS8IXJil6FCIAr47NxcSgNe3dkCSuFibqBj2CcjrCy7M6jQiZpex6RlF17ihWpSX6HGmxY0vG2xyl0MhYtgnoD2NVlgcftwxIR1pav4IKfq+NScHKoWAt3Z2wuUJyF0OhYBJkWAkScJnhywQBeDOydxuSbGRm67CwvJMdNv9+GBft9zlUAgY9gnmZJMLl9o9mFaiR266Su5yKIUsLM9EtlGJTw6a0drNvjmJhmGfYD49FOxGePfUTJkroVSjUYl4uCoHPj/wxg72zUk0DPsE0trtxZGzDows0KBkMLdbUuzNGK3HmMI0HD7rQN0F9s1JJAz7BLLpsAUSgLun8hAVyUMQBDxSzb45iYhhnyDsLj92HO9BtkGBaSW87SDJp8ikwdyJ6Wjp8mIz++YkDIZ9gthRb4XbK+GuKRlQKniIiuT1wLW+Oe+xb07CYNgnAJ9fwqbDFqiVAu6YwO2WJL+v9s1Zv4t9cxIBwz4B7Gm0odPqQ9V4I/RpCrnLIQLw1745O+rZNycRMOzjXECS8MG+bihEYME0brek+KEQBXznDvbNSRQM+zh38LQdLV1e3D7OyENUFHdKi9g3J1Ew7OOYJEl4f183BACLynlVT/Hpet+cv+zshJN9c+IWwz6OHbvgxMU2D8pH6zEoWy13OUQ3dL1vjtnuxwd72TcnXjHs49j71xpO3TsjS+ZKiG5tYXkmcoxKfHqIfXPiFcM+Tp1scuJUswuTRuhQZNLIXQ7RLWlUIr41J9g3Z9129s2JRwz7OHX9r8O8qqdEUV6ix7ihaThyzoGDZ+xyl0N/g2Efh85ddeHYRSfGDU1jwzNKGIIg4HvVJijE4FZMt5eLtfGEYR+HPtgbbGO8eDqv6imxFOaosWBaJjqtPmzYzcXaeMKwjzPnW904eNaOkYM0KCvSyl0O0YDdOyMLuenBxdqmDrfc5dA1DPs4886uTgDAg5XZEAQ2PKPEo1GJ+F51LvwB4L+2dCDAk7VxQRnOk6urq6HX6yGKIhQKBd59991I1ZWSGpudqLsQnKsvK9LJXQ5RyCYX63HbKD0OnrGj9rgVVePZwE9uYYU9ALz66qvIzs6ORC0pTZIkrK8Ndg98sDJH5mqIwvfdubmov+jAn3d0YspIPYxaNvGTE6dx4kTdBQcar+2r5w4cSgY5RiXur8iGzRXAn3d0yl1Oygv7yv6xxx6DIAhYvnw5li9ffss/m5Wlg1KZHL/dTSZjxD6XPyDhnXXNEATgicVDYTLFZmHWYLz14pnBmPyHuTjGG4vU6/vbNQbsPe3AzuNW3D3dhNtGR2c6J5Lvx2QVVti/+eabyM/PR2dnJ1auXIni4mKUl5ff9M93dyfHDYpNJiPa260R+3zbj/XgQqsLs8uMMCh9Ef3ct2Kz3jzsDUbNLT+eDDjGm4vka3DlvBz87zea8OL6S3j+e0Oh1UR2QiHS78d4FIlfZmF91/Pz8wEAOTk5uOuuu1BXVxd2QanG7Q3gnS+7oFYKeOB2rn1Q8hmWp8Gi8ix0WX2czpFRyGHvcDhgs9l6/3vXrl0oKSmJWGGp4pMDZpjtftxzWwayjWHPqhHFpSUzszA0V42tx3pQfzE5/oafaEIO+87OTnz729/Gvffei2XLlmHOnDmoqqqKZG1Jr6PHiw/3m5GhU2DhNJ6WpeSlVAh4/O48iALwn5+3s++9DEK+lBw6dCjef//9SNaSct7c3gmPT8KKedkRn8ckijcj8jVYWJ6JD/aZ8dbOTqyYZ5K7pJTChJHJ8UsO7D8dbItQWcqdBJQals7MRmGOCluO9uDEJU7nxBLDXgY+v4TXt3ZAAPDI3FyIbItAKUKlFPB4TXA65z8+aYPV6Ze7pJTBsJfBxwfMaO704o4J6Sgu4AEqSi0jB6XhgcpsdNv9+L+ftUFi75yYYNjH2NVuDzbu6UaGXoGHZnOrJaWmheWZKCvS4sg5BzYdtshdTkpg2MeQJEl4ZXM7vH4Jj8zNhT4tOU4TEw2UKAh44p48GLUi/ryjExdak/twWzxg2MfQ9norGi67MKVYh/ISvdzlEMkq06DEE/fkwx8A/v2jVm7HjDKGfYy0W7x4Y1sHdBoR35tnYq96IgATR+iwYFomWs1evLalXe5ykhrDPgYCAQlrP22DyxucvsnhSVmiXg9WZqO4QINdDTZsreuRu5ykxbCPgU8PWdDY7EJ5iR63jzPIXQ5RXFEqBPxgYT70aSJe+6Idp6+45C4pKTHso+xsiwtv13YiQ6fACk7fEN2QKUOFHy7KhyQB//rBVXRZfXKXlHQY9lFkd/nx7x+1IhAAnpifh3Qdd98Q3UxZkQ4Pz8mBxe7H796/CreXC7aRxLCPEkmS8PKmdnT0+LBkZhbGD+M9ZYn6cveUDFSVGXGh1Y0/fNSKQIAHriKFYR8lH+434+AZO8YNScPSmexoSdQfgiDg0TtNKCvS4vA5B17f1sETthHCsI+CQ2ftWF/bhWyjEt9fmA9R5Dw9UX8pFQJ+tDgfQ3LU2HykB+/vNctdUlJg2EfY5XY3/vRxK1RKAT9ZUoAMPbdZEg2UTqPAU/cPQm66Eu982cWWChHAsI+gjh4v/s97LXB5Jay6Jw/D85L/ZtZE0ZJtVOLpBwYjQ6/A61s7sP0Y9+CHg2EfIVanH//yTgu6bX48XJWD6aO5n54oXPlZKjz9wCAY0kT8v8/b8UUdr/BDxbCPALvLj9++24KWbi/m35aBBdMy5S6JKGkMydXg2WWDYdSK+K/NHfjsEOfwQ8GwD5Pd5cdv3mnBuVY3ZpcZsbwqR+6SiJLOUJMGP3+oEBl6BdZt68TbtZ3cpTNADPswWJ1+vLD+Cs5fC/rH7jLxrlNEUVKYo8YvlhciP1OFD/aZsfbTNvj8DPz+YtiHqM3sxZo3m3GhzYM54414rMbELZZEUZaXqcI+lUwvAAAKZElEQVQvHi7sbZz2/NtX0GX1yl1WQmDYh6Dxsh1r/tyMVrMXi8ozsZJX9EQxk65T4NllgzFjjAGnr7jw5L+dwhk2T+sTw34AJEnC1roePPUfZ2B1+rFiXi4emp3DoCeKMY1KxPcX5GH57Gx0Wb349VvN+GBfN9sr3AJP/PSTyxPAf2/twM7jVhi1CjxxTx4mjmC/GyK5CIKAheVZmDwmC7958wLeru1C/QUHHqvJQ16mSu7y4g6v7PuhsdmJ//Xfl7HzuBXD8zV46UejGfREcWLySCN+9b2hmDpSh4YmF37+2mV8csAMP6/yv4ZX9rdgd/nx7pfd2HwkeJBj4bRM3H97NvKzNGhv98hcHRFdZ9Qq8ON7C7D7pA3rtnXgzR2d2Hncim/fkcOOs9cw7G/AH5BQe9yKt3d1ocfhR0GWCv/j7jyUDE6TuzQiuglBEHD7OCMmDNfhrZ2d2FFvxW/eacHE4Trcf3sWigtS+/3LsP+KQEDCnkYb3tvdjVazF2qlgGWzsnHP1EyolFyEJUoERq0Cj9XkYd6kDLyxvQN1Fxyou+DA5OLgzc3HFKal5B3jGPYAHG4/dtRbsemwBR09PihEYO7EdCyZkYVs3hycKCENzw+2WThx2Yl3v+zGkXMOHDnnwLA8NeZNysCM0QZoNamzbJmySebzSzjZ5ETtcSsOnLHD45OgVgqYOzEdC6dlcjWfKAkIgoCyIh1Kh2px6ooLmw5ZcOCMHf/5eTte39qB20bpMXWkHhOGa6HTJPdtQ1Mq7Dt6vDjV7MKxiw4cOeuA3R28x2V+pgpV4424Y0I6jNrk/oETpSJBEDCmUIsxhVp0Wn2oPW7FzuM92H3Sht0nbVCIwJghWkwp1mH8MB0GZauS7vxM0oa93eXHlS4vmjo8ONXsRGOzCx09f71jfZZBgYpx6Zg5xoCSwak5h0eUinKMSiyZmYV7Z2TiYrsHR87acficAycuOXHikhNAJ7RqESMKNBhZoMHIQWkYkqNGbroyoVuihBX2O3bswK9//WsEAgEsW7YMq1atilRdNyRJErw+CS6vBLc3AKvTD4vdD7PdD7PdB7Pdj6vdHlzp8sJi93/tufo0EVNH6jBmiBZjh2gxLE+ddL+5iaj/BEHA8DwNhudpsLQiG902H46cc+DUFSfOtbi/Ev5BChHIy1AhP0uF/EwVMvUKpOuu/6OEUStCoxKhUQpQKYW4u4AMOez9fj/WrFmDV155Bfn5+XjwwQdRXV2NUaNGRbI+AMCnh8x478tuuDwB9OeYRG66EhOHazEoW43CHDVGDUrD4Jzk+2sZEUVOlkGJuRPTMXdiOoDg7MC5q26cb3XjSpcHrd1eXDV70dLdd+M1AYBaJUClECCKAtJUAn60uADDZLx7XchhX1dXh2HDhmHo0KEAgIULF2LLli1RCXtDmgL5WSqolcFvmkYV/A2qTxORqVcgU69Ehl6BTL0CpgwVNKrUWWEnoujQpykwYbgOE4Z//VCWzelHm8ULi8OPHrs/+G+HHzanH25fcNbB45Pg8Qbg9UsISIBaKUAh8xRQyGHf2tqKgoKC3v/Pz89HXV3dLZ9jMhlD+lr3zTHivjmDQ3putIQ6lnjy0LzEHwMRENv3ownAiJh9tcgJ+RL4RneJibc5KiIiCgo57AsKCnD16tXe/29tbUVeXl5EiiIiosgKOewnTJiACxcu4PLly/B4PPjoo49QXV0dydqIiChCQp6zVyqV+OUvf4nHH38cfr8fDzzwAEpKSiJZGxERRYgg8RbtRERJj3sUiYhSAMOeiCgFMOy/YseOHbj77rtx1113Ye3atd/4+P79+3HfffehtLQUn376ae/jDQ0NWL58ORYuXIjFixfj448/jmXZAxLqGK+z2WyYPXs21qxZE4tyQxLOGK9cuYK/+7u/w/z587FgwQI0NTXFquwBCWeMv/nNb7Bw4ULMnz8fv/rVr264jToe9DXGV155BQsWLMDixYuxYsUKNDc3937svffeQ01NDWpqavDee+/FsuwBCXWMIWWORJIkSZLP55PmzZsnXbp0SXK73dLixYul06dPf+3PXL58WWpoaJB++tOfSp988knv4+fOnZPOnz8vSZIkXb16VaqsrJQsFkssy++XcMZ43T/+4z9Kq1evlv7hH/4hVmUPSLhj/O53vyvV1tZKkiRJNptNcjgcMau9v8IZ48GDB6Xly5dLPp9P8vl80kMPPSTt2bMn1kPoU3/GuHv37t6fz7p166Qf//jHkiRJUnd3t1RdXS11d3dLZrNZqq6ulsxmc8zH0JdwxhhK5vDK/pqvtn9Qq9W97R++asiQIRg7dixE8evfthEjRmD48OEAgieJs7Oz0dXVFavS+y2cMQJAfX09Ojs7UVlZGauSByycMZ45cwY+n693fHq9HlqtNma191c4YxQEAR6PB16vt/ffubm5sSy/X/ozxpkzZ/b+fCZPntx77qe2thaVlZXIzMxERkYGKisrsXPnzpiPoS/hjDGUzGHYX3Oj9g+tra0D/jx1dXXwer0oKiqKZHkREc4YA4EAXnjhBfzsZz+LVnkREc4YL1y4gPT0dPzwhz/E0qVL8cILL8Dv9/f9xBgLZ4xTpkzBjBkzMGvWLMyaNQuzZ8/GyJEjo1VqyAY6xvXr16Oqqiqk58olnDF+VX8zh2F/jRSB9g9tbW346U9/iueff/6GV8ZyC2eMb7zxBqqqqjBo0KBIlxVR4YzR5/PhwIEDePrpp7F+/Xo0NTXh3XffjXSJYQtnjBcvXsTZs2exfft27NixA3v27MH+/fsjXWLYBjLGjRs3or6+Ho8//viAnyuncMZ43UAyJ2lvXjJQ4bZ/sNlseOKJJ/CTn/wEkydPjkaJYQtnjIcPH8bBgwfx5ptvwm63w+v1QqfT4amnnopWuSEJZ4wFBQUoLS3t7eQ6b948HD16NCp1hiOcMX7++eeYNGkS9Ho9AGD27Nk4cuQIysvLo1JrqPo7xi+//BJ/+tOf8Prrr0OtVvc+d9++fV977vTp06Nf9ACFM0Zg4JkTf5efMgmn/YPH48EPfvADLFmyBPPnz49ypaELZ4y//e1vsW3bNnzxxRd4+umnsXTp0rgLeiC8MU6YMAEWi6V37nPv3r1RadkdrnDGOHjwYOzfvx8+nw9erxf79++Py2mc/ozxxIkT+OUvf4k//vGPyMnJ6X181qxZqK2thcVigcViQW1tLWbNmhXrIfQpnDGGlDkRXFxOeNu2bZNqamqkefPmSX/4wx8kSZKk3/3ud9LmzZslSZKko0ePSrNnz5YmTZokTZ8+XVqwYIEkSZK0YcMGqbS0VLr33nt7/zlx4oRs47iVUMf4Ve+8807c7saRpPDGWFtbKy1atEhatGiR9PTTT0tut1uWMfQl1DH6fD7pF7/4hXTPPfdI8+fPl/7pn/5JtjH0pa8xrlixQqqoqOh9zz3xxBO9z3377belO++8U7rzzjul9evXy1J/f4Q6xlAyh+0SiIhSAKdxiIhSAMOeiCgFMOyJiFIAw56IKAUw7ImIUgDDnogoBTDsiYhSwP8HF4+Oufu/hOcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(all_stores_smape)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=10, minmax=(0.13758328818926077, 0.18497158371207476), mean=0.15876830385920487, variance=0.0002354032604925673, skewness=0.3271973487275648, kurtosis=-0.9587751430852118)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "describe(all_stores_smape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a model for each item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>item</th>\n",
       "      <th colspan=\"6\" halign=\"left\">1</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"6\" halign=\"left\">50</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>store</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>...</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2013-01-01</th>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>19</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>45</td>\n",
       "      <td>36</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-02</th>\n",
       "      <td>11</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>25</td>\n",
       "      <td>23</td>\n",
       "      <td>30</td>\n",
       "      <td>54</td>\n",
       "      <td>44</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-03</th>\n",
       "      <td>14</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>28</td>\n",
       "      <td>38</td>\n",
       "      <td>20</td>\n",
       "      <td>54</td>\n",
       "      <td>29</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-04</th>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>27</td>\n",
       "      <td>52</td>\n",
       "      <td>43</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-05</th>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>22</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>31</td>\n",
       "      <td>33</td>\n",
       "      <td>18</td>\n",
       "      <td>48</td>\n",
       "      <td>53</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 500 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "item        1                      ...  50                    \n",
       "store       1   2   3   4   5   6  ...  5   6   7   8   9   10\n",
       "date                               ...                        \n",
       "2013-01-01  13  12  19  10  11  20 ...  19  20  21  45  36  33\n",
       "2013-01-02  11  16   8  12   9   6 ...  25  23  30  54  44  37\n",
       "2013-01-03  14  16  10   8  12  11 ...  28  38  20  54  29  46\n",
       "2013-01-04  13  20  15  15   8   7 ...  27  33  27  52  43  51\n",
       "2013-01-05  10  16  22  19  13  12 ...  31  33  18  48  53  41\n",
       "\n",
       "[5 rows x 500 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items_time_series = pd.pivot_table(\n",
    "    columns=['item','store'], values='sales',\n",
    "    index=store_item_data.index, data=store_item_data).asfreq('D')\n",
    "items_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.2043320424047193\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 15, BIC -> 6, FPE -> 15, HQIC -> 6>\n",
      "Using 15 based on aic criterion\n",
      "0.13445202689996147\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.15741923536156543\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.20120284034167052\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.24970152598833356\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.12942933421786357\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.12863479551418702\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.11866561463768463\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.1376883394260311\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.11284699217169543\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.12012753168700632\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.11787332996262459\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.11491647120370825\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.13187597914707128\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.1079554425439\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.19855118250488368\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.1723317493868881\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.11344391017057162\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.15193098277727066\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.14166204101891786\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.16095980004545482\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.11093603642335355\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.1808236193490169\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 16, BIC -> 6, FPE -> 16, HQIC -> 6>\n",
      "Using 16 based on aic criterion\n",
      "0.12227766031454101\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.10351964176964461\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.14792959942507267\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.20894285395589313\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 7>\n",
      "Using 13 based on aic criterion\n",
      "0.1029443135882081\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.12582718288542172\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.15963237418646978\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.13219985642766646\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.15052725137817607\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.12255504894529416\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.19002541903165365\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.12379437224104943\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.11272356435197321\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.1868260323414077\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.11319047108255897\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.15519510165703648\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.1815004771798632\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.2247066627129281\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.16505144674459196\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.1431511638352087\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.17806507145740066\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.11436285813577104\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.12649667664699574\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.21582488166098793\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 14, BIC -> 6, FPE -> 14, HQIC -> 6>\n",
      "Using 14 based on aic criterion\n",
      "0.13822003310461012\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.18572490790042442\n",
      "<statsmodels.tsa.vector_ar.var_model.LagOrderResults object. Selected orders are: AIC -> 13, BIC -> 6, FPE -> 13, HQIC -> 6>\n",
      "Using 13 based on aic criterion\n",
      "0.12751262851051898\n"
     ]
    }
   ],
   "source": [
    "all_items_smape = evaluate_multiple_var(items_time_series, number_of_items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f9ff097f908>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4lOW9PvD7nT2zZJ8kZGXfCYiAIIs1QECSACK4HDnHo6dVz68VLdVT0eppbT1W66G9ziJK9bhLFVqkgKglgOz7voMJ2SCTkEySyezL+/sjkgISksySmXfm/lwXlzDJO/k+vpM7T555FkEURRFERCQ5snAXQERE/mGAExFJFAOciEiiGOBERBLFACcikihFT36x+nqL39cmJWlhNtuCWE1ki7X2ArHXZrY3+gWrzUaj4YaPS6YHrlDIw11Cj4q19gKx12a2N/qFus2SCXAiIrpWpwG+ZMkSTJgwAcXFxdc8/uGHH2LGjBkoKirCa6+9FrICiYjoxjodA583bx4WLlyIn//85+2P7d69G6WlpVi7di1UKhUaGhpCWiQREX1fpz3wsWPHIiEh4ZrHVqxYgUcffRQqlQoAkJKSEprqiIioQ37NQrlw4QL279+P3//+91Cr1fi3f/s35Ofnd3pdUpI2oEH9jt6JjVax1l4g9trM9ka/ULbZrwD3er1oaWnBZ599hmPHjuGpp55CaWkpBEG46XWBTKcxGg0BTUOUmlhrLxB7bWZ7o1+w2hzUaYTp6emYPn06BEFAfn4+ZDIZzGZzQAUSEVH3+BXg06ZNw+7duwEA5eXlcLvdSEpKCmphRER0c50OoSxevBh79+6F2WzGlClT8MQTT+Cee+7Bc889h+LiYiiVSvz2t7/tdPiEiIiCq9MAX7p06Q0ff/3114NeDHVs89GWcJfQ7s78+HCXQETgSkwiIsligBMRSRQDnIhIohjgREQSxQAnIpIoBjgRkUQxwImIJIoBTkQkUQxwIiKJYoATEUkUA5yISKIY4EREEsUAJyKSKAY4EZFEMcCJiCSKAU5EJFEMcCIiiWKAExFJVKcBvmTJEkyYMAHFxcXf+9g777yDQYMGobGxMSTFERFRxzoN8Hnz5uHtt9/+3uOXLl3Czp07kZmZGZLCiIjo5joN8LFjxyIhIeF7j7/yyit45plneBo9EVGYdHoq/Y2UlpYiLS0NgwcP7tZ1SUlaKBRyf74kAMBoNPh9rRRd3V69wRnGSq4VyvsQy/c4FsRae4HQtrnbAW632/Hmm2/i//7v/7r9xcxmW7evucJoNKC+3uL39VJzfXtbLZET4KG6D7F+j6NdrLUXCF6bO/oh0O1ZKJWVlaiursacOXNQUFCA2tpazJs3D/X19QEXSUREXdftHvigQYOwa9eu9n8XFBRg1apVSE5ODmphRER0c532wBcvXoz7778f5eXlmDJlClauXNkTdRERUSc67YEvXbr0ph/ftGlT0IohIqKu40pMIiKJYoATEUkUA5yISKIY4EREEsUAJyKSKAY4EZFEMcCJiCSKAU5EJFEMcCIiiWKAExFJFAOciEiiGOBERBLFACcikigGOBGRRDHAiYgkigFORCRRDHAiIonq9ESeJUuWYMuWLUhJScG6desAAK+++io2b94MpVKJ3NxcvPLKK4iPjw95sURE9Hed9sDnzZuHt99++5rHJk6ciHXr1mHt2rXo3bs33nrrrZAVSEREN9ZpgI8dOxYJCQnXPDZp0iQoFG2d91GjRqG2tjY01RERUYc6HULpzJ///GfcddddXfrcpCQtFAq531/LaDT4fa0UXd1evcEZxkquFcr7EMv3OBbEWnuB0LY5oABftmwZ5HI5Zs+e3aXPN5ttfn8to9GA+nqL39dLzfXtbbVEToCH6j7E+j2OdrHWXiB4be7oh4DfAb569Wps2bIF7733HgRB8LswIiLyj18BvnXrVvzxj3/ERx99hLi4uGDXREREXdBpgC9evBh79+6F2WzGlClT8MQTT2D58uVwuVx4+OGHAQAjR47ESy+9FPJiiYjo7zoN8KVLl37vsQULFoSkGCIi6rqAZ6FQ9PCJIlqsXthdPiTqFIhTc6EuUSRjgBPMrR7sP29FrdkFj/fvj2tUAvr30mBEnhYaFcOcKNIwwGOY2yvi4HkrTlXbIYpAok6O1Pi2nnez1Yv6ZjeOV9hxptqB/D5ajMiL44wjogjCAI9Rbo+Ivx1uhqnJDUOcDLcN0iMnVX3N53i8Is7U2HH0gg0HzltxucWNycO45w1RpGCAxyC39+/h3TtNjcnDDFDIv9+zVsgFDMvVol+GBpuPtqCizgWLrQm3DdQhQceXDlG4cWAzxoiiiM1Hr4S3CncMv3F4X02jkqFwdAIGZmnQ2OrB66svwe709VDFRNQRBniMOVFpR02DG1kpKtwxPB4yWdfGtOUyAbcP1mNglgYVdS784a+X4PIwxInCiQEeQxosHhw4b0WcSsCUYYYuh/cVgiBgwiA9bu2nw6kqB975uh6iKIaoWiLqDAM8Rni8IrYeb4FPBCYNNfg9LVAmE/CvRWno10uNXadbsflYS5ArJaKuYoDHiJNVdjRZvRicrUH2dbNNukulkOEnxRnQaWT4eHMDLtRFzk6JRLGEAR4DHC4fjpbboFYKGN1PF5TnTDEo8PjMNLi9Iv5nbS0cLo6HE/U0BngMOFxmhdsrYlQfHdTK4N3ykX11KBqTiLpmDz7b1hC05yWirmGAR7lmqwenaxyI18oxKFsT9Oefd3syslKU2HikBaeq7EF/fiLqGAM8yh0qs0EUgTH9dZB3c9ZJVygVAn40Iw2CALz9dR2cbg6lEPUUBngUs9i8uGByIlkvR65RFbKv0zdDg6Ixiahv9mDVjsaQfR0iuhYDPIqdqLRBBDC8tzbkm1DNnZCEtAQF/naoGZX1nJVC1BMY4FHK4fLh7EUHdBoZ+qQFNm2wK1QKGf6pwAifCLxfehk+LvAhCjkGeJQ6XW2H1wcMy43r9opLf+X30WLsAB3OXXRg24nYOn2cKBw6DfAlS5ZgwoQJKC4ubn+sqakJDz/8MAoLC/Hwww+jubk5pEVS93h9Ik5V2aFSCBiY2bOHTj/4g1SolQI+29YAq8Pb+QVE5LdOA3zevHl4++23r3ls+fLlmDBhAr7++mtMmDABy5cvD1mB1H0VdU443CIGZmmgVPTsAQzJBgVm35YEi92Hv+4x9+jXJoo1nQb42LFjkZCQcM1jpaWlmDt3LgBg7ty52LhxY2iqI7+cqXEAAAZmBn/ed1fMGJ2A1HgFvj7UDJPZHZYaiGKBX7vyNzQ0IC0tDQCQlpaGxsauTR1LStJCoZD78yUBAEajwe9rpejq9uoNXZvZ0dTqRq3ZjaxUNbIygrNs/mZ1deSHRT78dkUFVu9txgsL+wT1uaMJ2xv9QtnmHj1WxWy2+X2t0WhAfX3svDF2fXtbLV0L8EPnWgEA/TJUXb6mu7pyH4ZkyDEgU4OdJ5qx9WAdhuR0PhYf6/c42sVae4HgtbmjHwJ+zUJJSUlBXV0dAKCurg7Jycn+V0ZB4/WJOH/JAbVSQJ4x9FMHb0YQBPzDHSkAgE++uQyfj9MKiYLNrwAvKCjA559/DgD4/PPPMXXq1KAWRf6pqnfB4RLRL0PT6TFpPaFfLw1uH6JHRZ0L20/GVs+LqCd0GuCLFy/G/fffj/LyckyZMgUrV67Eo48+ih07dqCwsBA7duzAo48+2hO1Uie+rW1783JAVnjevLyReyelQKUQsHJHI+zccpYoqDodA1+6dOkNH3///feDXgz5z+n2ofqyC0l6OZL1kXNifLJBgVljEvH5bjPW7zNj/sSUcJdEFDW4EjNKXKhzwie2bSwVaYrGJiJJJ8eG/c1osHjCXQ5R1GCAR4my2rYZJ33Tw/vm5Y2olTLcMzEZbq+IVdt58ANRsDDAo4DV4UWt2Y30RCX0cf7Psw+lSUMNyDWqsONUK8pN3K2QKBgY4FGgvfedEXm97ytkMgEPfDet8E/fXIbI3QqJAsYAjwJlJicEAejdA9vGBmJYrhYj+2hxqtqBQ2X+L+oiojYMcImz2LxotHiQmayCRhX5t/P+KSmQCcCftjbA42UvnCgQkf8dTzd1oa5t+KR3WuiOTAumrBQVfpAfj1qzG5uPtoS7HCJJY4BLXEVd2/BJbpiXznfHvAlJ0KgErN7VyD3DiQLAAJewVocX9S0eZCQpJTF8ckW8VoGScUlodfiwdm9TuMshkizpfNfT91S0D59Ip/d9xYzRCUgxKPD1oSbUN3PPcCJ/MMAl7EqAS2n45AqVQoYFk5Lh8QKfbe/afvJEdC0GuETZnD6YmjxIT1RCq5bmbRw/WI8+6WrsOdOK8xcd4S6HSHKk+Z1PqL7c1vvOM0pj9smNyLhnOFFAGOASVXXZBQDIlnCAA8Cg7DiMHaDD+UtOlB7iIchE3cEAlyCvT8TFRjfitXIkaCNn61h/PXBH257h72y4yGmFRN3AAJcgk9kNj1dEdoq0e99XpMYrMWd8EpqtHvxlJ3vhRF3FAJeg9uGT1OgIcACYOToRWalqbDzSjMp67lZI1BUBBfh7772HoqIiFBcXY/HixXA6+Y3XE6obXFDIgYwkZbhLCRqlQsC/lmRBFIH3S7lbIVFX+D2AajKZ8MEHH+CLL76ARqPBk08+ifXr12PevHnBrI+u02zzoMXmRa5RBbksPAcXh2oPE71Bjbw0Fc5ddODtr+rRP7NrpwvdmR8fknqIIl1APXCv1wuHwwGPxwOHw4G0tLRg1UUdqP5u+CQnioZPrjZuoB4KGbDvfCucbh6CTHQzfvfA09PT8cgjj+DOO++EWq3GxIkTMWnSpJtek5SkhULh/4kxRqPB72ul6Or26g1tw1OXzG2934F5hog9fScQGUYtxg72YNfJFhyrcuAHI5M6vUbKrwsp1+6PWGsvENo2+x3gzc3NKC0tRWlpKQwGA5588kmsWbMGc+bM6fAas9n/TfyNRgPq6y1+Xy8117e31eKE2+NDzWUnkg0KwONBa5QdEKw3qNFqcWJAhgonL8hxrMyKnGQl0hNvPtYv1ddFrL+mY0Gw2tzRDwG/h1B27tyJ7OxsJCcnQ6lUorCwEIcOHfK7QOrcxUY3fGL0Dp9cIZcJmDi07QW746SFBz8QdcDvAM/MzMSRI0dgt9shiiJ27dqFfv36BbM2uk40Th/sSHqiEkNy4tBs8+JIOY9fI7oRv4dQRo4ciRkzZuDuu++GQqHAkCFDcN999wWzNrqKKIqovuyCWikgNV76qy+74tZ+OlTWO3Gswoa8NBVS46Nn2iRRMASUBIsWLcKiRYuCVQvdRKPFA7vLh34ZasiE8Ewf7GlKhYBJQw346mAzth63YPZtSVDIY6PtRF3BlZgSEUvDJ1fLTFa1D6XsP28NdzlEEYUBLhHVDS4IQtuhwLFmTH8dErRynKqyo6bBFe5yiCIGA1wCWmxe1Dd7kJaghFoZe7dMIRcwZbgBMgHYdqIFNicX+BABDHBJOHahbRZGrA2fXC01Xokx/XWwu0RsPdECH/dKIWKAS8Hh76bRRfv8784MzY1DTqoKlxrdOMqphUQM8Ejn9Yk4dsEGnUaGRF30LZ3vDkEQMHmYATq1DIfKbKi6zN0vKbYxwCPcuYsO2Jw+5KSqIMTI9MGbUStlKBgZD7kM2HrcgmZrdG0nQNQdDPAId6SM49/XS41X4vYhBrg8IkqPtsDm5DFsFJsY4BHucLkVSrmAXkkM8Kv176XBsNw4NFu9+K+1Ju6XQjGJAR7BLre4UdPgxtDcOK5AvIExA3TINapwstKOd/5Wx1N8KOYwwCPY4e+GT0b20Ya5ksgkEwTcMTwe/TLU2HGyFSu3N4a7JKIexQCPYFd24RvVlwHeEYVcwE/n9kJ6ohLr9jXhr3t4qj3FDgZ4hHK6fThZaUdWipK78HUiXivHswsykRqvwKodjdiwvyncJRH1CAZ4hDrybSvcXhGj+ujCXYokpBgUeHZ+JpL0cqzY2oC/7jFzTJyiHgM8Qu0703b25UgOn3RZWqISzy3IQoqhrSf+2bZGhjhFNQZ4BBJFEXtPN0OrlmFApibc5UhKepISv7g/CxlJSqzf34S3v67nFEOKWgzwCFTT4EZdkxsjemshl3H6YHelGBT4xX2Z6JOuxrYTFvzuLxdhdXCxD0WfgAK8paUFixYtwsyZM3HXXXfxUOMgOVLednABpw/6L16rwHP3ZmJMfx1OVTnwy09qUFXPvVMougQU4C+//DImT56ML7/8EmvWrOGhxkFyuMwGQQDyezPAA6FWyvCTknQUj02EqcmNX62owc5TlnCXRRQ0fgd4a2sr9u3bh/nz5wMAVCoV4uPjg1ZYrLI6vDh30YGB2VrEa2N798FgkAkC7p2cgidnZ0AuE/Dmhjq8tcHEIRWKCn4HeFVVFZKTk7FkyRLMnTsXzz//PGw27tEcqGMVdvhEYNwg/jAMplv76/DLB7PQN12NHada8dwHVe0HZRBJlSD6Oc/q2LFjuO+++7BixQqMHDkSv/nNb6DX6/HUU091eI3H44VCwV7lzbz+WQVKD5nx3z8ZiP5Zfx9C+WLv5TBWFdlmjUvt8ud6vSI+3WLCJ5tq4fUBRbel4F/uykScmq9Lkh6FvxdmZGQgIyMDI0eOBADMnDkTy5cvv+k1ZrP/PR6j0YD6+ugev/T52qYPJurk6JcZd017Wy3R/wac3qD2q53dfV1Mz9dhQHo23vrShPV7GrD3dDMW3pmKW/r27KKpWHhNXy3W2gsEr81Go+GGj/s9hGI0GpGRkYGysjIAwK5du/gmZoDKap2w2H0Y2UfLwxtCrHe6Gi89mIOisYloaPHg95/XYunnl2Bqcoe7NKIu87sHDgAvvPACnn76abjdbuTk5OCVV14JVl0x6crZl5w+2DOUCgH3TU7BxCEGfLi5HofLbDhRUYVZYxJRPC4RaiWXSVBkCyjAhwwZgr/85S/BqiXmHSm3Qi4DhuUxwHtSdqoKz87PxJ6zVqz45jLW7DFj+0kL7pucjNsG6fnbEEUsdjEihLnVg4o6FwZnxyFOxdvS0wRBwPhBerz6z7koGpuIZpsHb3xRh998ehHfXnKEuzyiG2JSRIj2wxu4eVVYaVQy3Dc5Bb99KBdjB+hw7qIDv1pRg7c2mNBo4QHKFFkCGkKh4DlU1rZ8fnQPz4SIBpuPtoTkeYfnaZEar8Ces1bsONWKPWdaMaK3FsPztB0ecXdnPufvU89hDzwCON0+nKiwIztFhbREHt4QSTKSVCgZl4iJQ/RQKgQcKrPhzzsb8W2tg1vVUtgxwCPA8Qo73F4Rt/Tj8EkkkgkCBmbF4Z7bk5HfWwun24etxy1Yv78Jl1s47ZDChwEeAQ59+93wST8On0QypUKGW/vrcPeEZPROV6O+2YN1e5uw92zb6UlEPY1j4GHm84k4VGZFglaOPhnqcJdDXWCIk+POEfG4mOXCzlMWnKi0o6LOiQmD9eEujWIMe+Bh9u13qy9H9dVCxvnGkpKZrMLc8cnI7x0Hq9OHvx1uwRvrTbDYudMh9QwGeJhx+ETaFHIBt/bXY/ZtSTDGK7D7TCt+8UEVTlRyp0MKPQZ4mB0qs0KlEDA0Ny7cpVAAkvUKzBqbiAWTktFi9+K1VZfwp60NPI+TQooBHkYmsxs1DW4My4vjvhtRQCYIKBmXhBfuz0JaohJf7G/Cr1ZU42KjK9ylUZRiaoTRQS7eiUp9MzT49cJs3DHcgIo6F/7942rsPdsa7rIoCjHAw+jQt1YIAEZx+XzU0ahk+JfCNPykOB0CgP9ZZ8LK7Q3w+TikQsHDAA8Ti92LszUO9O2lRoKOszmj1biBerz4QDbSE5VYu7cJv19Ty/M4KWgY4GFytNwGn8jZJ7EgO1WFX/5DFvJ7x+FIuQ2//KQGNQ0cF6fAMcDD5MrmVT19jBeFh04jx+K5vVA8NhGmJjd+/acaHCvnuDgFhgEeBm6PiKMXbDAmKJCVws2rYoVMJuDeySl4/K40ON0+PPfOt3xzkwLCAA+D4xU2OFwixvTX8bSXGHT7EAN+dncvKOUC/nedCV8fag53SSRRAQe41+vF3Llz8dhjjwWjnpiw91xbr2vcQO6dEauG52nx2mP9Ea+T46PNl/Hptgb4uD0tdVPAAf7BBx/wNPpucHtEHPrWhmSDAn25eVVM65+pxYv3ZyEjSYn1+5rwztf1nGZI3RJQgNfW1mLLli2YP39+sOqJeicrbbA5fRg7gMMnBBgTlHjh/iz0TVdj2wkL3vqyDl6GOHVRQBOQ/+M//gPPPPMMrFZrlz4/KUkLhULu99czGg1+Xxspjn5jBgAUjkuD0XjzGShXt1dvcIa0rkihN0j7t5LuvkaNRgOMAF573IAX3yvDrtOtkCnk+Pl9eVAqou8tqmj4Hu6uULbZ7wDfvHkzkpOTMXz4cOzZs6dL15jN/u/QZjQaUF9v8fv6SODxith5vAlJejlS4rw3bc/17W21RH+A6w1qybezO6/R6+/xUyVp+P3nl7DjeDP+3XYePy5OhyqKQjwavoe7K1ht7uiHgN+vjoMHD2LTpk0oKCjA4sWLsXv3bjz99NN+FxgLTlbaYXX6MHaAnnt/0/doVDIsvrsXhufF4VCZDX9YUwuX2xfusiiC+R3gP/vZz7B161Zs2rQJS5cuxfjx4/H6668Hs7aos+tM20/icQO5eIduTK2U4ak5GRjZR4vjFXb84a+1cHkY4nRj0fP7WYRzun04cM6K1HgFBmRqwl0ORTCVQoYnZ2dgVN+2EP/vtSbuK043FJQAv+222/DWW28F46mi1uEyGxxuERMG6zn7hDqlkAv4SXE6hue17Z/yv+sZ4vR97IH3kJ2n24ZPJgyOvXfhyT8qhQxPzc7A0Jw4HDhvxZsbTJxiSNdggPeAVrsXR8ttyElVITtVFe5ySEJUShl+OjcDg7I02HvWij9+VcfFPtSOG1H3gL3nWuH1AbcP4dL5aLf5aEuXP1dvcHZ52uTYATo0tnqw81Qr6ps9mDika0Nxd+bHd7kekh72wHvAzlNte5+M5/AJ+UmpkKHwlgSkxitw7qIDu063QuTeKTGPAR5il8wunK1xYGhOHFIM/IWH/Kf6LsST9QqcqXFgz1krQzzGMcBDbNvxtjcvpwxn75sCp1bKMGN0AhJ1cpyqsmP/eYZ4LGOAh5DXJ2L7SQu0ahnG9OfiHQoOjUqGmaMTkaCV43iFHYfL/N+igqSNAR5CR8ttaLJ6MWGwHiol/1dT8MSp23rihjgZDpfbcKS8axvKUXRhqoTQN8fbZiTcMZwzASj4dBo5Zt6aCL1GhoPf2nC8gj3xWMMAD5EmqweHy2zINarQO13aW6RS5NJr5Jg5OhFatQz7zllxqsoe7pKoBzHAQ2TL0Rb4ROAHI9j7ptAyaOWYOToBcSoBu8+04mwNQzxWMMBDwOMVseloC+JUMkwaytknFHoJOgVmjE6EWilgx6lWnL/oCHdJ1AMY4CGw/1wrmqxeTB5mgEbF/8XUM5L0CswcnQiVQsC2kxacY4hHPaZLCPztcNubl9NGcfiEelayQYGZoxOgVgrYftLSraX9JD0M8CC7YHLi3EUH8ntrkZHEjauo56XEKzFzdCI0SgHvbqzHxsPN4S6JQoQBHmRfHmwCAEy/hb1vCp9kgwJ33dq22OeDTZfx5YGmcJdEIcAAD6L6Zjd2n25FVooSI3prw10OxbhEvQJL7s1Eok6OT75pwF/3mLnsPsr4HeCXLl3CP/7jP+Kuu+5CUVER3n///WDWJUnr9zfBJwIl45J4aDFFhMxkFZ67NxMpBgVW7WjEh5sucz/xKOJ3gMvlcjz77LPYsGEDPv30U3zyySc4f/58MGuTlCarB9uOW2BMUOC2Qdz3myJHRpIKL9yfhewUFTYeacH/rjfxoOQo4XeAp6WlYdiwYQAAvV6Pvn37wmQyBa0wqfnqQDPcXhFFYxIhl7H3TZEl2aDA8/dlYlCWBvvOWfGff7kEm9Mb7rIoQIIYhEGx6upqLFy4EOvWrYNe33Hv0+PxQqGQB/rlIk5TqweP/O4kNGoZ3ntmaEg2rvpi7+WgPydFv1njUq/5t8vtw2ufVmDHiWb0ydDgV//cF8YEzpaSqoBPGLBarVi0aBGee+65m4Y3AJjN/m+2YzQaUF9v8fv6UPpo82XYXT7cMzEZzU3B2RXu+vZ29egtKdMb1DHRzit6or03+p750fQUaBQiSo+04Cf/dQZPFKdjUHZcSOsAIvt7OFSC1Waj8cYrugPqKrrdbixatAglJSUoLCwM5Kkkq67JjdIjzUhLUKCA5w+SBMhkAv6pIBUL70xFq92L3666iI2HmzlDRYL8DnBRFPH888+jb9++ePjhh4NZk6T8eWcjvD5g/sRkKOQc+yZpEAQBhbck4Nn5mdCqZfhg02W8/XU939yUGL8D/MCBA1izZg12796NOXPmYM6cOfjmm2+CWVvEKzc5set0K3qnqzGOM09IggbnxOGlhTnok67GthMWvPzpRZjM7nCXRV3k9xj4mDFjcObMmWDWIik+n4j3N9YDAO6fnMx53yRZKQYFnr83E+9vuoxtJyx4/sMq/MMdKbgzPx4CX9cRjSsx/bT5WAvKTE5MGKzH0FyuuiRpUyll+NGMNPy/WWlQygW8V3oZ/7n6EsytnnCXRjfBAPdDk9WDldsboVXL8MAdKeEuhyhoxg824OV/ysGIvDgcvWDHcx9UYddpC9/gjFAM8G4SRREfbb4Mm9OHBZOSkagLeCYmUURJNijw9LxeeGhqKtweEcu+qMMrKy+isj52pnhKBdOnm7aftGDvWSsGZGpwJ49LowgXyH7gMkFAybgk7D3bitPVDrzwYTUGZmkwup/Or4NK7p3K06mCjQHeDSazGx9uuow4lQyP35UGGZfMU5SL18oxbVQCahpc2HOmFWdqHCg3OZHfW4vB2XFQKvg9EE4cQukij1fEsg0mONwiHpqaCmOCMtwlEfWYrBQV5o5PwriBOgDA/vNWrNzRgMNlVjjdnDseLuyBd4Eoinh3Yz3Kap2YOESP24fwV0GKPTKZgGG5WvTvpcGpKjtOVNpxqMyG4xV2DMmJw5CM8ZS6AAAKyUlEQVScOGjV7BP2JAZ4F6zf14RtJyzok67GP08zhrscorBSK2UY1VeHYblxOF3twPFKG45esOFYhQ15aWoMztIgI0nJOeQ9gAHeiT1nWvHZ9kYk6+X46ZwMqEOw0yCRFCkVMozorcWQnDicu+jA6Wo7LpicuGByIkErx6AsDfpmaBDHXnnIMMBvYu/ZViz7wgSNUsBP5/ZCop7/u4iup5ALGJITh8HZGtQ1e3Cm2o5ykxN7z1mx75wVvZKV6JuhgdXB/ceDjYnUgd1nWvHmFyaolAKemZeJvDR1uEsiimiCICA9UYn0RCXGDfTh21oHymuduNjoxsVGN/a8fBz5vbW4tb8OI/toYYiLvrMBehoD/DqiKGLdvias2t4IjUqGZ+b1Qv9MTbjLIpIUjUqGYblaDMvVosXmRbnJgXqLFwfOW3HgvBWCAAzM1GB0fx1G99UhPYmzuvzBAL+K0+3Du3+rx87TrUjWy/HUnF7onc6eN1Eg4rVyjOyjw4KCTBw504iD31px6FsrztY4cKbGgRXfNCArRYlb+uoworcWAzI13Jq5ixjg3zlbY8cfv6qHqcmN/r3UWDQ7g8vkiYJIEARkpaiQlaJCybgkNFk9OFJmw8FvrTheYce6fU1Yt68JaqWAwdlxGJ4Xh+F5WmQmc0ZLR2I+oVpsXny+qxGlR9qWHM8ak4h5tydBpeA750ShlKhT4I4R8bhjRDycbh9OVtlxosKOYxU2HClv+wM0IEkvx/A8LQZnazAwKw5pCQoG+ndiNsBb7V6UHmnB+v1mOFwi0hOV+NEMIwZmhf5sQCK6llopwy19dbilb9tKzwaLB8crbDhRYceJShu2nbBg24m2syUTdXIMzNJgUFYcBmVrkJ2iitltLWIqwEVRxIU6F7451oLtJy1weUQY4mSYf2cKCvLjOe5GFCFSDArcMTwedwyPh08UUVnvwtlqO87UOHC2xoG9Z63Ye7btAHGtWoY+6er2P73T1UiNj41eetQHuMvjw7mLDhyvsGP/OStMTW3HRaXGKzB9VAJ+MCKeCw2IIphMENA7TY3eaWoUjm7riNU1eXCm5u+BfqKybWn/FXqNDDmpKmR+N+aendL2d0OcLKqCPaAA37p1K15++WX4fD4sWLAAjz76aLDq6jafKMJi86K2yY1LjW5U1btQZnKgss4Ft7dtM3qVQsD4QXqMH6zHyD5ayGP01y4iKRMEAelJSqQnKTFleNuWzlaHFxV1TpSb2v5cMDlxutqBU9WOa67VqASkJShhTFAiLUGB1HglEnVyJOoVSNTJkaCTS+r9L78D3Ov14qWXXsK7776L9PR0zJ8/HwUFBejfv38w6wMANFo82Fd2GfWNdjhcPthdPjhcPrTafTBbPWhq9aDZ5oX3uk3R5DIgO1WFITlxGJYbh8HZcVwKTxSFdBo5huZqrzne0On24VKjGzUNLlQ3uHCx0YXLzR6YmtyorHd1+FwalQCtWg6dWgatWgatpu2/OrUcGpUApVwGhRxQygUoFAKU8rY/8quHYL87wCje5EVLsx1qpQxDc+OCPkzrd4AfPXoUeXl5yMnJAQAUFRWhtLQ0JAG+elcjvjluueHHFPK2d7P7pKuRqFMgPVGJjGQlslNUyElVQcXAJopJaqUMvb8bE7+aKIqw2L2oa/LgssWDJqsHza3ets6g1YtWuxc2pw8NFg+qLgdvq9yfzsnALf10QXs+IIAAN5lMyMjIaP93eno6jh49etNrjEb/tmF99kEDnvXrSmm7+v8XTzOhaOBvBgRbGoB+eeGuInB+d09vdMhpNL05QEQU6fwO8IyMDNTW1rb/22QyIS0tLShFERFR5/wO8BEjRuDChQuoqqqCy+XC+vXrUVBQEMzaiIjoJvweA1coFHjxxRfxwx/+EF6vF/fccw8GDBgQzNqIiOgmBPFGg9lERBTxOMeOiEiiGOBERBIV9gDfunUrZsyYgenTp2P58uXf+7jL5cJTTz2F6dOnY8GCBaiurgYAVFdXIz8/H3PmzMGcOXPw4osv9nTpfuuszfv27cPdd9+NoUOH4ssvv7zmY6tXr0ZhYSEKCwuxevXqnio5IIG0d8iQIe33+PHHH++pkgPWWZvfffddzJo1CyUlJXjooYdQU1PT/rFovMc3a2+03uMVK1agpKQEc+bMwQMPPIDz58+3f+ytt97C9OnTMWPGDGzbts3/IsQw8ng84tSpU8XKykrR6XSKJSUl4rlz5675nI8++kh84YUXRFEUxXXr1olPPvmkKIqiWFVVJRYVFfV4zYHqSpurqqrEU6dOic8884y4YcOG9sfNZrNYUFAgms1msampSSwoKBCbmpp6ugndEkh7RVEUR40a1ZPlBkVX2rxr1y7RZrOJoiiKH3/8cfvrOlrvcUftFcXovccWi6X97xs3bhQfeeQRURRF8dy5c2JJSYnodDrFyspKcerUqaLH4/GrjrD2wK9ejq9SqdqX419t06ZNuPvuuwEAM2bMwK5du264iEgqutLm7OxsDB48GDLZtbdn+/btmDhxIhITE5GQkICJEycG9tO7BwTSXqnqSpvHjx+PuLi2vedHjRrVvqYiWu9xR+2Vqq60Wa/Xt//dbre3L3QsLS1FUVERVCoVcnJykJeX1+kq9o6E9TvmRsvxTSbT9z6nV69eANqmLhoMBpjNZgBtwyhz587FwoULsX///p4rPABdaXMorg2XQGt2Op2YN28e7r33XmzcuDEUJQZdd9u8atUqTJkyxa9rI0Eg7QWi+x5//PHHmDZtGn73u9/hF7/4Rbeu7Yqw7gd+o5709cvxO/qctLQ0bN68GUlJSTh+/Dh+/OMfY/369df81ItEXWlzKK4Nl0Br3rx5M9LT01FVVYWHHnoIAwcORG5ubjBLDLrutHnNmjU4fvw4Pvroo25fGykCaS8Q3ff4wQcfxIMPPoi1a9di2bJlePXVV4N6j8PaA+/KcvyMjAxcunQJAODxeGCxWJCYmAiVSoWkpCQAwPDhw5Gbm4vy8vKeK95PgWxBIMXtCwKtOT09HQCQk5ODcePG4eTJk0GvMdi62uadO3fizTffxLJly6BSqbp1bSQJpL1AdN/jK4qKitp/uwjmPQ5rgHdlOX5BQUH7O/FfffUVxo8fD0EQ0NjYCK/XCwCoqqrChQsX2re2jWSBbEEwadIkbN++Hc3NzWhubsb27dsxadKkEFccmEDa29zcDJerbd/mxsZGHDx4MCTbFQdbV9p88uRJvPjii1i2bBlSUlLaH4/We9xRe6P5Hl+4cKH971u2bEFeXtv2hwUFBVi/fj1cLld7duXn5/tXiF9vfQbRli1bxMLCQnHq1KniG2+8IYqiKP7hD38QN27cKIqiKDocDvGJJ54Qp02bJt5zzz1iZWWlKIqi+OWXX4qzZs0SS0pKxLlz54qlpaVha0N3ddbmI0eOiJMnTxZHjhwpjhs3Tpw1a1b7tStXrhSnTZsmTps2TVy1alVY6u8uf9t74MABsbi4WCwpKRGLi4vFzz77LGxt6K7O2vzQQw+JEyZMEGfPni3Onj1bfOyxx9qvjcZ73FF7o/ke//rXvxZnzZolzp49W1y4cKF49uzZ9mvfeOMNcerUqWJhYaG4ZcsWv2vgUnoiIomKjnlbREQxiAFORCRRDHAiIoligBMRSRQDnIhIohjgREQSxQAnIpKo/w/2sSkc4lw0RAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(all_items_smape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=50, minmax=(0.1029443135882081, 0.24970152598833356), mean=0.14912976749311493, variance=0.0013015468133833437, skewness=0.8241553410554039, kurtosis=-0.1881847700143413)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "describe(all_items_smape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf]",
   "language": "python",
   "name": "conda-env-tf-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}