07-kernel_linear_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a72be7eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import plotly.express as px\n",
    "import math\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from scripts.linear_experiments import linear_test_model, LPM_LS\n",
    "from scripts.utils import score_models, plot_predictions, save_fig\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (10, 4)\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "font = {\n",
    "    'weight' : 'regular',\n",
    "    'family': 'serif',\n",
    "    'size'   : 16\n",
    "}\n",
    "plt.rc('font', **font)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6a6b096",
   "metadata": {},
   "outputs": [],
   "source": [
    "LIST = ['ERA5', 'METEO-FRANCE']\n",
    "WEATHER_SOURCE = LIST[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8980b43",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(f'data/features/features-{WEATHER_SOURCE}_per_vineyard.csv', index_col=0, header=[0,1], dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "315d734a",
   "metadata": {},
   "outputs": [],
   "source": [
    "VINEYARDS = list(df_test.T.index.get_level_values(0).unique())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7cfe58b5",
   "metadata": {},
   "source": [
    "# Transform prices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f803774",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictors = ['WD: flowering - harvest', 'DTR: véraison - harvest', 'P: flowering']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd3cd16c",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_dfs = []\n",
    "for vineyard in VINEYARDS:\n",
    "    df_extract = df_test.loc[:, vineyard]\n",
    "    df_extract = df_extract[['0 - Price']+predictors]\n",
    "    df_extract.loc[:, 'Vintage'] = df_extract.index\n",
    "    df_extract.loc[:, \"Vineyard\"] = vineyard\n",
    "    list_dfs.append(df_extract)\n",
    "df_extract = pd.concat(list_dfs, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74636106",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_extract[\"Vintage\"] = df_extract[\"Vintage\"]-1955"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f2c94d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.scatter_3d(\n",
    "    df_extract,\n",
    "    x=predictors[0],\n",
    "    y=predictors[1],\n",
    "    z=predictors[2],\n",
    "    color='0 - Price',\n",
    "    size=\"Vintage\",\n",
    "    size_max=25,\n",
    "    opacity=0.7,\n",
    "    hover_data=predictors+[\"Vintage\"]\n",
    ")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a06b2d44",
   "metadata": {},
   "outputs": [],
   "source": [
    "first_vintage = 1960\n",
    "last_vintage = 2022\n",
    "target_variable = '0 - Price'\n",
    "predictors = ['P: flowering', 'WD: flowering - harvest', \"DTR: véraison - harvest\"]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "190404a2",
   "metadata": {},
   "source": [
    "# LLS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76bde6df",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def kernel(n, i, j, h):\n",
    "    return 1\n",
    "\n",
    "\n",
    "model = LPM_LS(kernel, degree=0)\n",
    "df_results_ols, coeffs = linear_test_model(\n",
    "    VINEYARDS,\n",
    "    model,\n",
    "    \"Classical OLS\",\n",
    "    df_test,\n",
    "    target_variable,\n",
    "    predictors,\n",
    "    first_vintage,\n",
    "    last_vintage,\n",
    "    plot=False,\n",
    "    fixed_bandwidth=1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21a11db4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "BANDWIDTH = 0.375\n",
    "\n",
    "\n",
    "def gaussian_kernel(n, i, j, h):\n",
    "    return np.exp(-np.square(np.abs(i - j) / (n * h)))\n",
    "\n",
    "\n",
    "gaussian_model = LPM_LS(gaussian_kernel, degree=0)\n",
    "df_results_lpm_k1, coeffs = linear_test_model(\n",
    "    VINEYARDS,\n",
    "    gaussian_model,\n",
    "    \"LLS gaussian kernel\",\n",
    "    df_test,\n",
    "    target_variable,\n",
    "    predictors,\n",
    "    first_vintage,\n",
    "    last_vintage,\n",
    "    plot=False,\n",
    "    fixed_bandwidth=BANDWIDTH,\n",
    ")\n",
    "\n",
    "\n",
    "def exponential_kernel(n, i, j, h):\n",
    "    return np.exp(-np.abs(i - j) / (n * h))\n",
    "\n",
    "\n",
    "exponential_model = LPM_LS(exponential_kernel, degree=0)\n",
    "df_results_lpm_k1_bis, coeffs = linear_test_model(\n",
    "    VINEYARDS,\n",
    "    exponential_model,\n",
    "    \"LLS exponential kernel\",\n",
    "    df_test,\n",
    "    target_variable,\n",
    "    predictors,\n",
    "    first_vintage,\n",
    "    last_vintage,\n",
    "    plot=False,\n",
    "    fixed_bandwidth=BANDWIDTH,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcfa8742",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results_lpm_k1_bis.to_csv('model_outputs/prices_predicted.csv')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "150324c8",
   "metadata": {},
   "source": [
    "### Flat 3d bar plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a317e6b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_training(model, color, vintage_examples):\n",
    "    n_examples = len(vintage_examples)\n",
    "    fig, axes = plt.subplots(n_examples, figsize=(10, 2.5), sharex=True)\n",
    "    for test_vintage, offset, ax in zip(\n",
    "        vintage_examples,\n",
    "        np.linspace(0, 1, n_examples),\n",
    "        axes,\n",
    "    ):\n",
    "        xs = np.arange(1960, test_vintage + 1)\n",
    "        bar_heights = [1 for _ in range(len(xs))]\n",
    "        if model == \"LLS\":\n",
    "            print(\"OKOK\")\n",
    "            bar_heights = [\n",
    "                exponential_kernel(len(bar_heights), i, len(bar_heights), BANDWIDTH)\n",
    "                for i in range(len(bar_heights))\n",
    "            ]\n",
    "            print(bar_heights)\n",
    "        bar_heights[-1] = 0.05\n",
    "\n",
    "        cs = [color] * len(xs)\n",
    "        cs[-1] = \"r\"\n",
    "        ax.bar(xs, bar_heights, color=cs, alpha=0.8)\n",
    "\n",
    "        ax.set_ylabel(\"\")\n",
    "        ax.set_yticks([])\n",
    "        ax.spines[\"top\"].set_visible(False)\n",
    "        ax.spines[\"right\"].set_visible(False)\n",
    "        ax.spines[\"left\"].set_visible(False)\n",
    "        ax.set_title(\n",
    "            f\"Training for vintage {test_vintage}\", x=1.0, y=0.3, ha=\"left\", va=\"center\"\n",
    "        )\n",
    "\n",
    "    plt.xlabel(\"Vintage\")\n",
    "    ax.set_xticks(np.arange(1960, 2015, 1), minor=True)\n",
    "    ax.set_xticks(np.arange(1960, 2015, 5), minor=False)\n",
    "    plt.savefig(\n",
    "        f\"views/training/{model}.jpg\", dpi=300, bbox_inches=\"tight\", transparent=\"True\"\n",
    "    )\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "vintage_examples = [1981, 1994, 2013]\n",
    "plot_training(\"LLS\", \"g\", vintage_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d932652",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.lineplot(data=coeffs[\"Château Trotanoy\"][predictors])\n",
    "plt.axhline(0, color='k')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "d28154cd",
   "metadata": {},
   "source": [
    "# Display evolution of coeffs per region"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "131dbacf",
   "metadata": {},
   "outputs": [],
   "source": [
    "coeffs_df = pd.concat(coeffs.values(), keys=coeffs.keys()).reset_index()\n",
    "coeffs_df.columns = ['Vineyard', 'Year'] + list(coeffs_df.columns[2:])\n",
    "coeffs_df = coeffs_df.melt(id_vars=['Vineyard', 'Year'])\n",
    "coeffs_df = coeffs_df.loc[~(coeffs_df['variable'] == 'intercept')]\n",
    "coeffs_df = coeffs_df.loc[coeffs_df['Year'] >= 2010]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b35f399b",
   "metadata": {},
   "outputs": [],
   "source": [
    "prices = pd.read_excel('data/prices/prices_per_vineyard.xlsx', index_col=0)\n",
    "appellations = prices.groupby('Vineyard')['Appellation'].first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7808025f",
   "metadata": {},
   "outputs": [],
   "source": [
    "coeffs_df = coeffs_df.merge(appellations, on='Vineyard')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "536b9dd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "coeffs_df.groupby('Vineyard')['Appellation'].first().value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9066737d",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.box(coeffs_df, x='Year', y='value', color='Appellation', facet_col='variable', width=1200)\n",
    "fig.add_hline(y=0.0)\n",
    "fig.show()\n",
    "fig.write_html('views/coefficients.html')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "3a443260",
   "metadata": {},
   "source": [
    "# Compare vs Real Prices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb793ea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results_lpm_k1_bis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dac4460",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results = pd.concat(\n",
    "    [df_results_ols, df_results_lpm_k1, df_results_lpm_k1_bis], axis=0\n",
    ")\n",
    "df_prices = df_test.loc[:, (VINEYARDS, \"0 - Price\")].droplevel(1, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e44a2d9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = score_models(df_results, df_prices, 1994, 2013)\n",
    "\n",
    "models = scores.columns\n",
    "median_scores = scores.reset_index().groupby(\"Test variable\")[models].median()\n",
    "display(median_scores)\n",
    "median_scores.reset_index().to_excel(\n",
    "    f\"model_outputs/scores/lls_predictions-{WEATHER_SOURCE}_per_vineyard.xls\",\n",
    "    index=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60ed1a80",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "stats.ttest_ind(\n",
    "    scores.loc[\"MAE\", \"LLS exponential kernel\"],\n",
    "    scores.loc[\"MAE\", \"Classical OLS\"],\n",
    "    equal_var=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cddd002",
   "metadata": {},
   "outputs": [],
   "source": [
    "WA_rated_vineyards = [\n",
    "    'Château Angélus',\n",
    "    'Château Ausone',\n",
    "    'Château Cheval Blanc',\n",
    "    'Château Figeac',\n",
    "    'Château Haut-Brion',\n",
    "    \"Château L'Evangile\",\n",
    "    'Château La Mission Haut-Brion',\n",
    "    'Château Lafite Rothschild',\n",
    "    'Château Léoville Las Cases',\n",
    "    'Château Margaux',\n",
    "    'Château Montrose',\n",
    "    'Château Mouton Rothschild',\n",
    "    'Château Palmer',\n",
    "    'Château Pape Clément',\n",
    "    'Château Pavie',\n",
    "    'Château Trotanoy',\n",
    "    'Château Trotte Vieille',\n",
    "    'Petrus',\n",
    "    'Vieux Château Certan',\n",
    "    # 'Château Latour' # useless to add since we don't have recent prices\n",
    "]\n",
    "\n",
    "spearman = scores.loc[\"Spearman\"]\n",
    "spearman = spearman[spearman.index.isin(WA_rated_vineyards)]\n",
    "spearman.median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ec8b95d",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores.reset_index().to_excel(f'model_outputs/scores/lls_predictions-{WEATHER_SOURCE}_per_vineyard.xlsx', index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "60c63e66",
   "metadata": {},
   "source": [
    "# Compare recent prices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09c6c22f",
   "metadata": {},
   "outputs": [],
   "source": [
    "START_YEAR = 2009"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b56d784",
   "metadata": {},
   "outputs": [],
   "source": [
    "vineyards_best_perf = scores.loc['Spearman', 'LLS gaussian kernel'] #.nlargest(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63866175",
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_prices = df_results.loc[\"LLS gaussian kernel\"].loc[\n",
    "    vineyards_best_perf.index, START_YEAR:\n",
    "]\n",
    "current_prices = df_prices.T.loc[vineyards_best_perf.index, START_YEAR:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5fc26d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "current_prices = current_prices.reset_index().melt(\n",
    "    id_vars=\"Vineyard\", var_name=\"Year\", value_name=\"Log price\"\n",
    ")\n",
    "current_prices[\"source\"] = \"Prix Idealwine, avril 2023\"\n",
    "predicted_prices = predicted_prices.reset_index().melt(\n",
    "    id_vars=\"Vineyard\", var_name=\"Year\", value_name=\"Log price\"\n",
    ")\n",
    "predicted_prices[\"source\"] = \"Prix de long-terme prédit\"\n",
    "complete_df = pd.concat([current_prices, predicted_prices])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "fbbd3a63",
   "metadata": {},
   "source": [
    "### Display true vs predicted prices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8230d62e",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.scatter(\n",
    "    complete_df,\n",
    "    x=\"Year\",\n",
    "    y=\"Log price\",\n",
    "    color=\"source\",\n",
    "    facet_col=\"Vineyard\",\n",
    "    facet_col_wrap=3,\n",
    "    facet_row_spacing=0.02,\n",
    "    height=math.ceil(complete_df[\"Vineyard\"].nunique()/3) * 200,\n",
    "    width=1200,\n",
    ")\n",
    "fig = fig.update_xaxes(matches=None)\n",
    "fig = fig.update_yaxes(matches=None)\n",
    "fig.write_html(\"views/compare_recent_prices.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e44d44c",
   "metadata": {},
   "outputs": [],
   "source": [
    "difference = (\n",
    "    predicted_prices.groupby(['Vineyard', 'Year'])['Log price'].first()\n",
    "     - current_prices.groupby(['Vineyard', 'Year'])['Log price'].first()\n",
    ")\n",
    "difference = difference.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23326075",
   "metadata": {},
   "outputs": [],
   "source": [
    "difference = difference.dropna(how='any', axis=0)\n",
    "difference['Year']= difference['Year'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "736a0ced",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "plt.figure(figsize=(12, 6))\n",
    "plt.axhline(0, color='k')\n",
    "sns.boxplot(data=difference, x='Year', y='Log price', color='grey')\n",
    "plt.ylabel('Surestimation du prix de long-terme')\n",
    "plt.tight_layout()\n",
    "plt.savefig('views/avg_prediction_error.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7934c71",
   "metadata": {},
   "outputs": [],
   "source": [
    "complete_df = complete_df.merge(appellations, on='Vineyard')\n",
    "difference = difference.merge(appellations, on='Vineyard')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73569d1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.box(complete_df, x='Year', y='Log price', color=\"source\", facet_col='Appellation', facet_col_wrap=2, width=900)\n",
    "fig.write_html('views/price_average.html')\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a4b06a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "longterm = complete_df.loc[complete_df['source'] == 'Prix de long-terme prédit']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47f8df48",
   "metadata": {},
   "outputs": [],
   "source": [
    "longterm['Log price'] = pd.to_numeric(longterm['Log price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2434c8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_year = longterm.groupby(['Vineyard']).apply(lambda group: group.loc[group['Log price'].idxmax(), ['Year', 'Appellation']]).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9bfea78",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_year.loc[top_year['Appellation']=='Médoc', 'Year'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "87fe25ff",
   "metadata": {},
   "source": [
    "# Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d21ca976",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plot_predictions(\n",
    "    df_results.rename(index={\"LLS gaussian kernel\":\"Local Least Squares\"}),\n",
    "    df_prices,\n",
    "    1994,\n",
    "    2013,\n",
    "    \"Local Least Squares\",\n",
    "    set_limits=False\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "9c09db28",
   "metadata": {},
   "source": [
    "## Plot overall fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b902f90",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results_lls = df_results.loc[\"LLS exponential kernel\", :].copy()\n",
    "df_results_lls = (\n",
    "    df_results_lls.reset_index()\n",
    "    .melt(id_vars=\"Vineyard\", var_name=\"Vintage\", value_name=\"Predicted price\")\n",
    "    .groupby([\"Vintage\", \"Vineyard\"])\n",
    "    .first()\n",
    ")\n",
    "df_prices_ex = (\n",
    "    df_prices.loc[1980:, :]\n",
    "    .reset_index()\n",
    "    .melt(id_vars=\"index\", var_name=\"Vineyard\", value_name=\"True price\")\n",
    "    .rename(columns={\"index\": \"Vintage\"})\n",
    "    .groupby([\"Vintage\", \"Vineyard\"])\n",
    "    .first()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42b959cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_prices_ex, df_results_lls = np.exp(df_prices_ex), np.exp(df_results_lls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96bec221",
   "metadata": {},
   "outputs": [],
   "source": [
    "concat=pd.concat([df_results_lls, df_prices_ex], axis=1).reset_index()\n",
    "concat.columns = [\"Vintage\", \"Vineyard\", \"Predicted price\", \"True price\"]\n",
    "concat = concat[concat[\"Vintage\"].isin(range(1994, 2014))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fe1cdc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "prices = pd.read_excel(\"data/prices/prices_04-2023.xlsx\", header=0, index_col=0)\n",
    "appellation = prices[[\"Appellation\"]]\n",
    "\n",
    "concat[\"Appellation\"] = concat.apply(\n",
    "    lambda row: appellation.loc[row[\"Vineyard\"]], axis=1\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "61c25a7b",
   "metadata": {},
   "source": [
    "### Fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "663c402b",
   "metadata": {},
   "outputs": [],
   "source": [
    "means = concat.groupby([\"Vineyard\", \"Appellation\"]).mean()\n",
    "concat_rescaled = concat.copy()\n",
    "concat_rescaled.loc[:, [\"Predicted price\", \"True price\"]] = concat_rescaled.apply(\n",
    "    lambda row: 100\n",
    "    * row[[\"Predicted price\", \"True price\"]]\n",
    "    / means.loc[(row[\"Vineyard\"], row[\"Appellation\"]), \"True price\"],\n",
    "    axis=1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77da4de7",
   "metadata": {},
   "outputs": [],
   "source": [
    "HUE_BY_VINEYARD = True\n",
    "RESCALED = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e22409c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import cm\n",
    "\n",
    "if HUE_BY_VINEYARD:\n",
    "    plt.figure(figsize=(5, 5))\n",
    "else:\n",
    "    plt.figure(figsize=(6, 5))\n",
    "plt.rcParams.update({\"font.size\": 16})\n",
    "plt.grid(False, which=\"both\", axis=\"both\")\n",
    "if not RESCALED:\n",
    "    plt.yscale(\"log\")\n",
    "    plt.xscale(\"log\")\n",
    "\n",
    "cmap = cm.get_cmap(\"coolwarm\", 27)\n",
    "\n",
    "ax = sns.scatterplot(\n",
    "    data=(concat_rescaled if RESCALED else concat),\n",
    "    x=\"True price\",\n",
    "    y=\"Predicted price\",\n",
    "    hue=(\"Vineyard\" if HUE_BY_VINEYARD else \"Vintage\"),\n",
    "    marker=\"s\",\n",
    "    edgecolor=\"k\",\n",
    "    legend=False,\n",
    "    palette=(cmap if not HUE_BY_VINEYARD else \"colorblind\"),\n",
    "    alpha=0.7,\n",
    "    zorder=2,\n",
    ")\n",
    "if RESCALED:\n",
    "    plt.ylabel(\"Prediction as % of average price\")\n",
    "    plt.xlabel(\"True price as % of average price\")\n",
    "else:\n",
    "    plt.ylabel(\"Predicted price (€)\")\n",
    "    plt.xlabel(\"True price (€)\")\n",
    "if not HUE_BY_VINEYARD:\n",
    "    norm = plt.Normalize(concat[\"Vintage\"].min(), concat[\"Vintage\"].max())\n",
    "    sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm)\n",
    "    sm.set_array([])\n",
    "    cbar = ax.figure.colorbar(sm)\n",
    "    cbar.set_ticks(range(1994, 2013, 5))\n",
    "if not RESCALED:\n",
    "    bottom_limit, top_limit = 30, 5000\n",
    "else:\n",
    "    bottom_limit, top_limit = 20, 170\n",
    "\n",
    "plt.plot([bottom_limit, top_limit], [bottom_limit, top_limit], color=\"k\", zorder=1)\n",
    "plt.ylim((bottom_limit, top_limit))\n",
    "plt.xlim((bottom_limit, top_limit))\n",
    "\n",
    "plt.gca().set_axisbelow(True)\n",
    "\n",
    "save_fig(\n",
    "    f\"views/model_outputs/global_fit_per_vineyard_{RESCALED=}_{HUE_BY_VINEYARD=}\",\n",
    "    width_column=\"1.5\",\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4337dbb2",
   "metadata": {},
   "source": [
    "### Rank fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcb876f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results_kls_rank = (\n",
    "    df_results.loc[\"LLS exponential kernel\", :].T.rank(ascending=False).T.copy()\n",
    ")\n",
    "df_results_kls_rank = (\n",
    "    df_results_kls_rank.reset_index()\n",
    "    .melt(id_vars=\"Vineyard\", var_name=\"Vintage\", value_name=\"Predicted price\")\n",
    "    .groupby([\"Vintage\", \"Vineyard\"])\n",
    "    .first()\n",
    ")\n",
    "df_prices_ex_rank = (\n",
    "    df_prices.loc[1980:, :]\n",
    "    .rank(ascending=False)\n",
    "    .reset_index()\n",
    "    .melt(id_vars=\"index\", var_name=\"Vineyard\", value_name=\"True price\")\n",
    "    .rename(columns={\"index\": \"Vintage\"})\n",
    "    .groupby([\"Vintage\", \"Vineyard\"])\n",
    "    .first()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "194b6969",
   "metadata": {},
   "outputs": [],
   "source": [
    "concat = pd.concat([df_results_kls_rank, df_prices_ex_rank], axis=1).reset_index()\n",
    "concat.columns = [\"Vintage\", \"Vineyard\", \"Predicted price\", \"True price\"]\n",
    "concat = concat[concat[\"Vintage\"] >= 1990]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3dd41e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(5, 4))\n",
    "cmap = cm.get_cmap(\"coolwarm\", 27)\n",
    "\n",
    "\n",
    "plt.grid(False, which=\"both\", axis=\"both\")\n",
    "ax = sns.scatterplot(\n",
    "    data=concat,\n",
    "    x=\"True price\",\n",
    "    y=\"Predicted price\",\n",
    "    hue=\"Vintage\",\n",
    "    marker=\"s\",\n",
    "    palette=cmap,\n",
    "    legend=False,\n",
    "    edgecolor=\"black\",\n",
    ")\n",
    "\n",
    "plt.ylabel(\"Predicted rank\")\n",
    "plt.xlabel(\"2021 rank\")\n",
    "norm = plt.Normalize(concat[\"Vintage\"].min(), concat[\"Vintage\"].max())\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm)\n",
    "sm.set_array([])\n",
    "\n",
    "ax.figure.colorbar(sm)\n",
    "\n",
    "plt.savefig(f\"views/model_outputs/global_rank_fit.png\", bbox_inches=\"tight\", dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "8845f7d6",
   "metadata": {},
   "source": [
    "---\n",
    "# End of notebook"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "0ac38357a05bc42f0ed259dcad053212499e7f4648ddbc814451284e59f63394"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}