cloud_orbit_demo_4_gpu_Vs_cpu.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d80c1bda-93e1-43e2-bdab-229a3879fb86",
   "metadata": {},
   "source": [
    "# -1) imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "25f09824-02ef-40ee-92ef-c4c9eb0a61af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start\n"
     ]
    }
   ],
   "source": [
    "print(\"start\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d5007f0a-8fe5-4916-972c-db15a5ffeb91",
   "metadata": {},
   "outputs": [],
   "source": [
    "from   io                       import BytesIO\n",
    "import pandas                   as     pd\n",
    "from   google.cloud             import storage\n",
    "from   sklearn.model_selection  import train_test_split\n",
    "from   sklearn.preprocessing    import StandardScaler, OneHotEncoder\n",
    "from   sklearn.impute           import SimpleImputer\n",
    "from   sklearn.compose          import ColumnTransformer\n",
    "from   sklearn.datasets         import fetch_openml\n",
    "from   sklearn.pipeline         import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eed82082-f4d7-4498-b0d4-1d9761415b62",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "import sklearn\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa762efa-f2c8-47d2-bdcd-d16118ef98f1",
   "metadata": {},
   "source": [
    "# 0) personnal function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d8b0f348-4dbb-47b4-894a-6591394aeaac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from : https://johaupt.github.io/scikit-learn/tutorial/python/data%20processing/ml%20pipeline/model%20interpretation/columnTransformer_feature_names.html \n",
    "def get_feature_names(column_transformer):\n",
    "    \"\"\"Get feature names from all transformers.\n",
    "    Returns\n",
    "    -------\n",
    "    feature_names : list of strings\n",
    "        Names of the features produced by transform.\n",
    "    \"\"\"\n",
    "    # Remove the internal helper function\n",
    "    #check_is_fitted(column_transformer)\n",
    "    \n",
    "    # Turn loopkup into function for better handling with pipeline later\n",
    "    def get_names(trans):\n",
    "        # >> Original get_feature_names() method\n",
    "        if trans == 'drop' or (\n",
    "                hasattr(column, '__len__') and not len(column)):\n",
    "            return []\n",
    "        if trans == 'passthrough':\n",
    "            if hasattr(column_transformer, '_df_columns'):\n",
    "                if ((not isinstance(column, slice))\n",
    "                        and all(isinstance(col, str) for col in column)):\n",
    "                    return column\n",
    "                else:\n",
    "                    return column_transformer._df_columns[column]\n",
    "            else:\n",
    "                indices = np.arange(column_transformer._n_features)\n",
    "                return ['x%d' % i for i in indices[column]]\n",
    "        if not hasattr(trans, 'get_feature_names'):\n",
    "        # >>> Change: Return input column names if no method avaiable\n",
    "            # Turn error into a warning\n",
    "            warnings.warn(\"Transformer %s (type %s) does not \"\n",
    "                                 \"provide get_feature_names. \"\n",
    "                                 \"Will return input column names if available\"\n",
    "                                 % (str(name), type(trans).__name__))\n",
    "            # For transformers without a get_features_names method, use the input\n",
    "            # names to the column transformer\n",
    "            if column is None:\n",
    "                return []\n",
    "            else:\n",
    "                return [name + \"__\" + f for f in column]\n",
    "\n",
    "        return [name + \"__\" + f for f in trans.get_feature_names()]\n",
    "    \n",
    "    ### Start of processing\n",
    "    feature_names = []\n",
    "    \n",
    "    # Allow transformers to be pipelines. Pipeline steps are named differently, so preprocessing is needed\n",
    "    if type(column_transformer) == sklearn.pipeline.Pipeline:\n",
    "        l_transformers = [(name, trans, None, None) for step, name, trans in column_transformer._iter()]\n",
    "    else:\n",
    "        # For column transformers, follow the original method\n",
    "        l_transformers = list(column_transformer._iter(fitted=True))\n",
    "    \n",
    "    \n",
    "    for name, trans, column, _ in l_transformers: \n",
    "        if type(trans) == sklearn.pipeline.Pipeline:\n",
    "            # Recursive call on pipeline\n",
    "            _names = get_feature_names(trans)\n",
    "            # if pipeline has no transformer that returns names\n",
    "            if len(_names)==0:\n",
    "                _names = [name + \"__\" + f for f in column]\n",
    "            feature_names.extend(_names)\n",
    "        else:\n",
    "            feature_names.extend(get_names(trans))\n",
    "    \n",
    "    return feature_names"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "204f417b-ea16-4294-a966-56a1ecfa3335",
   "metadata": {},
   "source": [
    "# 1) Fetch Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ddcd63c4-d2a9-4ecf-a531-d4ddc2ca8ab6",
   "metadata": {},
   "outputs": [],
   "source": [
    "USER_FLAG = \"--user\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75355d0d-17d1-4d0f-810d-29383a15b463",
   "metadata": {},
   "source": [
    "## 1.1) bucket configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "95ebbd7f-8ca5-45ed-ae70-1cb95e3dc191",
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_name  = \"cloud-orbit-bank-data\"\n",
    "file_name    = \"bank-data.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab37cfd1-0090-446a-95a2-13c5f25280fe",
   "metadata": {},
   "source": [
    "## 1.2) retrieving file content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b6ded9ea-0a35-4f6d-9ab4-440951e39de7",
   "metadata": {},
   "outputs": [],
   "source": [
    "client       = storage.Client()\n",
    "bucket       = client.get_bucket(bucket_name)\n",
    "blob         = bucket.get_blob(file_name)\n",
    "blob_string  = blob.download_as_string()\n",
    "content      = BytesIO(blob_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6da6cb0-c7d0-477c-8e79-69301aea83af",
   "metadata": {},
   "source": [
    "## 1.3) creating a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aef019de-74c5-425b-be38-6250160bb836",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>58</td>\n",
       "      <td>44</td>\n",
       "      <td>33</td>\n",
       "      <td>47</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>job</th>\n",
       "      <td>management</td>\n",
       "      <td>technician</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>blue-collar</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>marital</th>\n",
       "      <td>married</td>\n",
       "      <td>single</td>\n",
       "      <td>married</td>\n",
       "      <td>married</td>\n",
       "      <td>single</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>education</th>\n",
       "      <td>tertiary</td>\n",
       "      <td>secondary</td>\n",
       "      <td>secondary</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>default</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>balance</th>\n",
       "      <td>2143</td>\n",
       "      <td>29</td>\n",
       "      <td>2</td>\n",
       "      <td>1506</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>housing</th>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>loan</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>contact</th>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>day</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <td>may</td>\n",
       "      <td>may</td>\n",
       "      <td>may</td>\n",
       "      <td>may</td>\n",
       "      <td>may</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>duration</th>\n",
       "      <td>261</td>\n",
       "      <td>151</td>\n",
       "      <td>76</td>\n",
       "      <td>92</td>\n",
       "      <td>198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>campaign</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pdays</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>previous</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>poutcome</th>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>y</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0           1             2            3        4\n",
       "age                58          44            33           47       33\n",
       "job        management  technician  entrepreneur  blue-collar  unknown\n",
       "marital       married      single       married      married   single\n",
       "education    tertiary   secondary     secondary      unknown  unknown\n",
       "default            no          no            no           no       no\n",
       "balance          2143          29             2         1506        1\n",
       "housing           yes         yes           yes          yes       no\n",
       "loan               no          no           yes           no       no\n",
       "contact       unknown     unknown       unknown      unknown  unknown\n",
       "day                 5           5             5            5        5\n",
       "month             may         may           may          may      may\n",
       "duration          261         151            76           92      198\n",
       "campaign            1           1             1            1        1\n",
       "pdays              -1          -1            -1           -1       -1\n",
       "previous            0           0             0            0        0\n",
       "poutcome      unknown     unknown       unknown      unknown  unknown\n",
       "y                  no          no            no           no       no"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df           = pd.read_csv(content, sep=\";\")\n",
    "df.head(5).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03d8e6fd-fd7c-4bda-b27c-1bdc1c49f02a",
   "metadata": {},
   "source": [
    "# 2) Data preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1df83be1-13e6-415b-85ef-9200545d8f56",
   "metadata": {},
   "source": [
    "## 2.1) Pre-processing steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2a52f321-1dd7-4b19-9c5a-e973ecc68cbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "target = \"y\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "88d84a60-3123-4991-99bc-b914b402e117",
   "metadata": {},
   "outputs": [],
   "source": [
    "column_y = df[target]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ffe302fb-2d2b-42ef-90f1-94c37f47fb01",
   "metadata": {},
   "outputs": [],
   "source": [
    "numeric_features        = df.drop(target, axis=1).select_dtypes(\"int64\").columns\n",
    "categorical_features    = df.drop(target, axis=1).select_dtypes(\"object\").columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9d4b68a1-2d79-42c8-ac7c-60ef396c4508",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes catégorielles\n",
    "categorical_transformer = OneHotEncoder(handle_unknown=\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d1d68ea9-1cc5-4540-9570-0a17c26de3d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes numériques\n",
    "missing_values          = SimpleImputer(strategy=\"median\")\n",
    "centrer_reduire         = StandardScaler()\n",
    "steps                   = [(\"missing_values\",missing_values ), (\"scaler\",centrer_reduire )]\n",
    "numeric_transformer     = Pipeline(steps=steps )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e2a2f53-df74-4070-aca4-abf4d88867b7",
   "metadata": {},
   "source": [
    "### 2.1.1) get dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1fcf10a8-7e4f-4143-bcdc-4933228e1456",
   "metadata": {},
   "outputs": [],
   "source": [
    "transformers = [ (\"numerical\"   , numeric_transformer    , numeric_features     ) ,\n",
    "                (\"categorical\" , categorical_transformer, categorical_features ) ]\n",
    "preprocessor = ColumnTransformer( transformers=transformers )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "73e74af0-1990-402b-9596-75b4add34c0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = preprocessor.fit(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3e76510e-827c-4c74-b1e7-c785544972dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies = preprocessor.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "17f69b87-793d-42f5-a1e5-bce49878c40d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Transformer missing_values (type SimpleImputer) does not provide get_feature_names. Will return input column names if available\n",
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Transformer scaler (type StandardScaler) does not provide get_feature_names. Will return input column names if available\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "dummies = pd.DataFrame(dummies, columns=get_feature_names(preprocessor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "79b60a2a-48fe-46f7-ab36-361c96764c1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies[target] = column_y\n",
    "\n",
    "dummies[target] = dummies[target].map({\"no\":0, \"yes\":1})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8f94a09-ff2d-4ce4-8f31-9849ba4f35b9",
   "metadata": {},
   "source": [
    "## 2.2) creating a validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "36305adf-9c78-48e0-b0e0-1698a6cb405b",
   "metadata": {},
   "outputs": [],
   "source": [
    "pct_validation  = 0.05\n",
    "nb_ligne        = dummies.shape[0]\n",
    "validation_size = int (nb_ligne * pct_validation)\n",
    "validation_set  = dummies.sample(validation_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0f5c69dd-ecdf-48e0-8ce4-1187f0645d8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies         = dummies.drop(validation_set.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "acf3442f-9ebf-4cb5-8b8d-11c98d118fa9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2260, 52)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "34514923-6f00-4fc8-b535-2e22f164b07e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42951, 52)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "cdb39611-5eec-41ad-882f-a392d679cfcc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['numerical__age', 'numerical__balance', 'numerical__day',\n",
       "       'numerical__duration', 'numerical__campaign', 'numerical__pdays',\n",
       "       'numerical__previous', 'categorical__x0_admin.',\n",
       "       'categorical__x0_blue-collar', 'categorical__x0_entrepreneur',\n",
       "       'categorical__x0_housemaid', 'categorical__x0_management',\n",
       "       'categorical__x0_retired', 'categorical__x0_self-employed',\n",
       "       'categorical__x0_services', 'categorical__x0_student',\n",
       "       'categorical__x0_technician', 'categorical__x0_unemployed',\n",
       "       'categorical__x0_unknown', 'categorical__x1_divorced',\n",
       "       'categorical__x1_married', 'categorical__x1_single',\n",
       "       'categorical__x2_primary', 'categorical__x2_secondary',\n",
       "       'categorical__x2_tertiary', 'categorical__x2_unknown',\n",
       "       'categorical__x3_no', 'categorical__x3_yes', 'categorical__x4_no',\n",
       "       'categorical__x4_yes', 'categorical__x5_no', 'categorical__x5_yes',\n",
       "       'categorical__x6_cellular', 'categorical__x6_telephone',\n",
       "       'categorical__x6_unknown', 'categorical__x7_apr', 'categorical__x7_aug',\n",
       "       'categorical__x7_dec', 'categorical__x7_feb', 'categorical__x7_jan',\n",
       "       'categorical__x7_jul', 'categorical__x7_jun', 'categorical__x7_mar',\n",
       "       'categorical__x7_may', 'categorical__x7_nov', 'categorical__x7_oct',\n",
       "       'categorical__x7_sep', 'categorical__x8_failure',\n",
       "       'categorical__x8_other', 'categorical__x8_success',\n",
       "       'categorical__x8_unknown', 'y'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c327d6de-bc2d-4717-8f81-2bfd19fb0f07",
   "metadata": {},
   "source": [
    "# 3) Modelisation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74b49ad6-7c33-4bcd-9ff7-c32740fdb349",
   "metadata": {},
   "source": [
    "## 3.1) create X et y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "53866036-466e-4eac-8d60-321b862a852f",
   "metadata": {},
   "outputs": [],
   "source": [
    "y      = dummies[target]\n",
    "X      = dummies.drop(target, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3f69b3a0-c7bd-46cb-bb3a-ae98c67018c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42951, 51)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c875ea9-689c-4a28-aea5-3d73832a15d0",
   "metadata": {},
   "source": [
    "## 3.2) test / train / validation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "491f6cf0-3cde-438d-b03d-0df729069321",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e16f448d-c3fa-406a-b9d8-0aa20e925504",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_validation    = validation_set.drop(target, axis=1)\n",
    "y_validation    = validation_set[target]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28b2b92f-f8a5-4e3f-b818-0c179d02986c",
   "metadata": {},
   "source": [
    "## 3.3) Modélisation naïve"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acb56a48-6ad9-4971-9469-f264d5f57f4f",
   "metadata": {},
   "source": [
    "### 3.3.1) Randomforest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ca15596b-86a9-48f4-b0d3-fb6cc338d237",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "algo    = RandomForestClassifier()\n",
    "modele  = algo.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e76e0252-e3a9-45a2-b5c7-c7e915ea0e9c",
   "metadata": {},
   "source": [
    "### 3.3.2) scoring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1664a065-220e-44a0-9ee0-83955fb8b4f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "42136f2b-1985-4f47-9c67-8e605c031cfe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6791824117533298"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test, modele.predict(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3411047b-225e-4523-83fa-25c787fb467d",
   "metadata": {},
   "source": [
    "## 3.4) Modélisation avec un grid_search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3f8d2ab-1d1d-4b0c-917f-66ccb9c92fc5",
   "metadata": {},
   "source": [
    "### 3.4.1) grid search "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "63db246d-e520-4646-9786-6d134ea41bd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5930e3cc-0e3b-4a9d-8402-e37bc74e6a55",
   "metadata": {
    "tags": [
     "3.4.1"
    ]
   },
   "outputs": [],
   "source": [
    "hyper_parameters = {\n",
    "        'bootstrap'                : [ ],\n",
    "        'ccp_alpha'                : [ ],\n",
    "        'class_weight'             : [ ],\n",
    "        'criterion'                : [ \"gini\", \"entropy\" ],\n",
    "        'max_depth'                : [ 10, 20 ],\n",
    "        'max_features'             : [ ],\n",
    "        'max_leaf_nodes'           : [ ],\n",
    "        'max_samples'              : [ ],\n",
    "        'min_impurity_decrease'    : [ ],\n",
    "        'min_samples_leaf'         : [ ],\n",
    "        'min_samples_split'        : [ ],\n",
    "        'min_weight_fraction_leaf' : [ ],\n",
    "        'n_estimators'             : [ 20, 30 ],\n",
    "        'n_jobs'                   : [ ],\n",
    "        'oob_score'                : [ ],\n",
    "        'random_state'             : [ ],\n",
    "        'verbose'                  : [ ],\n",
    "        'warm_start'               : [ ],\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "a4b052f2-1529-49a7-b8ba-4d657b750552",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search_parameters = { k:v  for k,v in hyper_parameters.items() if v  }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "010c546b-e06c-43f9-9f67-fbc7380604a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=RandomForestClassifier(), n_jobs=-1,\n",
       "             param_grid={'criterion': ['gini', 'entropy'],\n",
       "                         'max_depth': [10, 20], 'n_estimators': [20, 30]},\n",
       "             scoring='roc_auc')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid = GridSearchCV(RandomForestClassifier(), grid_search_parameters, n_jobs=-1, scoring='roc_auc')\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ac285cb2-272c-425b-b1b3-ad3ee50986a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kpi : roc_auc = 0.92 \n",
      " Best estimator : RandomForestClassifier(max_depth=20, n_estimators=30)\n"
     ]
    }
   ],
   "source": [
    "print (f\"kpi : {grid.scoring} = {grid.best_score_:.2f} \\n Best estimator : {grid.best_estimator_}\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "82cdb0f7-53e9-46d0-bc45-333f543518e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_random_forest = grid.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0a3bda4-1cac-4e01-bf2d-92f281cb6b20",
   "metadata": {},
   "source": [
    "# 4) installing a missing library"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a85ef85b-aadb-45cc-8f9e-a4b75e4b7125",
   "metadata": {},
   "source": [
    "## 4.1) catboost is not installed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f0c08b7c-66c2-4676-a953-eb5b1de79127",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy \n",
    "from catboost import CatBoostClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f98699f0-ccb7-4129-8fe9-9520e6956190",
   "metadata": {},
   "source": [
    "## Catboost can be installed with pip install"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "685b41cb-bd9a-4c63-80d8-55de9deb317d",
   "metadata": {},
   "source": [
    "## Catboost is installed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8703a72b-2ad2-4336-8b82-29c92629c6a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import CatBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "10114f3e-dff7-40ca-8e72-9ca27098888c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "374a7a08-f0d0-4279-b324-c1bfd785b45e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "cfdd3f84-a55b-49be-91b0-0f56bf82cd6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: ipywidgets in ./.local/lib/python3.7/site-packages (7.7.0)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.7/site-packages (from ipywidgets) (5.2.1.post0)\n",
      "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in ./.local/lib/python3.7/site-packages (from ipywidgets) (1.1.0)\n",
      "Requirement already satisfied: ipython-genutils~=0.2.0 in /opt/conda/lib/python3.7/site-packages (from ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: ipython>=4.0.0 in /opt/conda/lib/python3.7/site-packages (from ipywidgets) (7.33.0)\n",
      "Requirement already satisfied: widgetsnbextension~=3.6.0 in ./.local/lib/python3.7/site-packages (from ipywidgets) (3.6.0)\n",
      "Requirement already satisfied: ipykernel>=4.5.1 in /opt/conda/lib/python3.7/site-packages (from ipywidgets) (6.13.0)\n",
      "Requirement already satisfied: nbformat>=4.2.0 in /opt/conda/lib/python3.7/site-packages (from ipywidgets) (5.4.0)\n",
      "Requirement already satisfied: nest-asyncio in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets) (1.5.5)\n",
      "Requirement already satisfied: debugpy>=1.0 in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets) (1.6.0)\n",
      "Requirement already satisfied: jupyter-client>=6.1.12 in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets) (7.3.1)\n",
      "Requirement already satisfied: tornado>=6.1 in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets) (6.1)\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets) (5.9.1)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets) (21.3)\n",
      "Requirement already satisfied: matplotlib-inline>=0.1 in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.3)\n",
      "Requirement already satisfied: pygments in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0->ipywidgets) (2.12.0)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0->ipywidgets) (3.0.29)\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0->ipywidgets) (4.8.0)\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0->ipywidgets) (5.1.1)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0->ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0->ipywidgets) (0.7.5)\n",
      "Requirement already satisfied: setuptools>=18.5 in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0->ipywidgets) (59.8.0)\n",
      "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0->ipywidgets) (0.18.1)\n",
      "Requirement already satisfied: jupyter-core in /opt/conda/lib/python3.7/site-packages (from nbformat>=4.2.0->ipywidgets) (4.10.0)\n",
      "Requirement already satisfied: jsonschema>=2.6 in /opt/conda/lib/python3.7/site-packages (from nbformat>=4.2.0->ipywidgets) (4.5.1)\n",
      "Requirement already satisfied: fastjsonschema in /opt/conda/lib/python3.7/site-packages (from nbformat>=4.2.0->ipywidgets) (2.15.3)\n",
      "Requirement already satisfied: notebook>=4.4.1 in /opt/conda/lib/python3.7/site-packages (from widgetsnbextension~=3.6.0->ipywidgets) (6.4.11)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /opt/conda/lib/python3.7/site-packages (from jedi>=0.16->ipython>=4.0.0->ipywidgets) (0.8.3)\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from jsonschema>=2.6->nbformat>=4.2.0->ipywidgets) (4.11.4)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema>=2.6->nbformat>=4.2.0->ipywidgets) (21.4.0)\n",
      "Requirement already satisfied: importlib-resources>=1.4.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema>=2.6->nbformat>=4.2.0->ipywidgets) (5.7.1)\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from jsonschema>=2.6->nbformat>=4.2.0->ipywidgets) (4.2.0)\n",
      "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema>=2.6->nbformat>=4.2.0->ipywidgets) (0.18.1)\n",
      "Requirement already satisfied: entrypoints in /opt/conda/lib/python3.7/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (0.4)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.7/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (2.8.2)\n",
      "Requirement already satisfied: pyzmq>=22.3 in /opt/conda/lib/python3.7/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (23.0.0)\n",
      "Requirement already satisfied: nbconvert>=5 in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (6.5.0)\n",
      "Requirement already satisfied: prometheus-client in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.14.1)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.15.0)\n",
      "Requirement already satisfied: argon2-cffi in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.3.0)\n",
      "Requirement already satisfied: Send2Trash>=1.8.0 in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.8.0)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (3.1.2)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.7/site-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.7/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=4.0.0->ipywidgets) (0.2.5)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->ipykernel>=4.5.1->ipywidgets) (3.0.9)\n",
      "Requirement already satisfied: zipp>=3.1.0 in /opt/conda/lib/python3.7/site-packages (from importlib-resources>=1.4.0->jsonschema>=2.6->nbformat>=4.2.0->ipywidgets) (3.8.0)\n",
      "Requirement already satisfied: defusedxml in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.7.1)\n",
      "Requirement already satisfied: jupyterlab-pygments in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.2.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.1.1)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.5.0)\n",
      "Requirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.11.1)\n",
      "Requirement already satisfied: bleach in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (5.0.0)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.8.4)\n",
      "Requirement already satisfied: nbclient>=0.5.0 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.6.3)\n",
      "Requirement already satisfied: tinycss2 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.1.1)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.8.2->jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (1.16.0)\n",
      "Requirement already satisfied: argon2-cffi-bindings in /opt/conda/lib/python3.7/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.2.0)\n",
      "Requirement already satisfied: cffi>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.15.0)\n",
      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.7/site-packages (from beautifulsoup4->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.3.1)\n",
      "Requirement already satisfied: webencodings in /opt/conda/lib/python3.7/site-packages (from bleach->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.1)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.7/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.21)\n"
     ]
    }
   ],
   "source": [
    "!pip install ipywidgets --user"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b07e83c0-c95e-40a5-b8c3-6b0b19241e56",
   "metadata": {},
   "source": [
    "## 4.2) Using catboost"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96918c76-347a-4560-930d-3d40d5ec7bbe",
   "metadata": {},
   "source": [
    "### 4.2.1) with GPU "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c2954b2c-b6f5-4c15-a15c-48485b5266de",
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "88bc15bc-408f-4445-9779-37e289bb8670",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU -> duration = 29.9 sec. : auc = 0.72 %\n",
      "CPU -> duration = 6.8 sec. : auc = 0.72 %\n"
     ]
    }
   ],
   "source": [
    "for task_type in [\"GPU\",\"CPU\"]:\n",
    "    print(f\"{task_type}\", end=\"\") \n",
    "    start     = time()\n",
    "    algo      = CatBoostClassifier(learning_rate=1, depth=6, silent=True, task_type=task_type)\n",
    "    modele    = algo.fit(X_train, y_train)\n",
    "    auc_score = roc_auc_score(y_test, modele.predict(X_test))\n",
    "    end       = time()\n",
    "    duration  = end - start \n",
    "    print(f\" -> duration = {duration:.1f} sec. : auc = {auc_score :.2f} %\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bcc40b08-6786-4522-be2c-ae795f59360b",
   "metadata": {},
   "source": [
    "## 4.3) Catboost GPU is slower on small dataset, but faster in big ones"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a56caeb7-96ca-47d9-806f-a4e9fa49009d",
   "metadata": {},
   "source": [
    "### 4.3.1) on crée une grosse df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b4af8231-c2b5-4ad9-a86f-a446a0ea4530",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42951, 52)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "7ebd1b5d-4ee5-49ad-9c40-f168b551d92d",
   "metadata": {},
   "outputs": [],
   "source": [
    "big_df = dummies\n",
    "for i in range(10):\n",
    "    big_df = big_df.append(dummies, ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "275b3859-ad47-4f72-8b56-8a913bc5fdbe",
   "metadata": {},
   "source": [
    "### 4.3.2) on recrée les dummies et le X et y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "090da00a-79d3-4921-adec-66b7d1a8b7be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(472461, 52)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "big_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "640b3dab-8065-43e3-8a62-0864b7a4795e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "52"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(dummies.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "2d619f77-00f9-43aa-a56a-c529fdc2c9ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "target = \"y\"\n",
    "dummies = pd.get_dummies(big_df)\n",
    "X      = dummies.drop(target, axis=1)\n",
    "y      = dummies[target]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "208b7602-4296-441e-ac86-a0634986b687",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "984a3a26-c7d5-43bc-a763-4bbacf020608",
   "metadata": {},
   "source": [
    "### 4.3.3) on modélise sur GPU et CPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "cbb53469-af1e-4e7a-9686-cea047dbdd90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU -> duration = 37.2 sec. : auc = 0.73 %\n",
      "GPU -> duration = 7.4 sec. : auc = 0.73 %\n"
     ]
    }
   ],
   "source": [
    "task_types = [\"CPU\",\"GPU\"]\n",
    "for task_type in task_types :\n",
    "    print(f\"{task_type}\", end=\"\") \n",
    "    start     = time()\n",
    "    algo      = CatBoostClassifier(learning_rate=1, depth=6, silent=True, task_type=task_type)\n",
    "    modele    = algo.fit(X_train, y_train)\n",
    "    auc_score = roc_auc_score(y_validation, modele.predict(X_validation))\n",
    "    end       = time()\n",
    "    duration  = end - start \n",
    "    print(f\" -> duration = {duration:.1f} sec. : auc = {auc_score :.2f} %\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e6a1840-a073-4a33-ab35-20abc71efa78",
   "metadata": {},
   "source": [
    "### 4.3.4) on peut 'watch' le nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "5c487216-9e1e-46cf-aa00-1752f7ba6aef",
   "metadata": {},
   "outputs": [],
   "source": [
    "#watch  nvidia-smi --format=csv --query-gpu=power.draw,utilization.gpu,fan.speed,temperature.gpu"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc8ecdba-3ab4-4fb5-af50-64c8615439e1",
   "metadata": {},
   "source": [
    "# 5) Création d'un pipeline de traitement de la donnée"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6c770a5-0cf2-46dc-867b-954ea42d66f0",
   "metadata": {},
   "source": [
    "## 5.1) Création d'un pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "2ebbd587-7e10-4b8c-bfab-888b6fc3f746",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1ecdfc56-1a01-4740-bc39-bee3d53b71b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "steps = [(\"preprocessor\", preprocessor) , (\"prediction\", modele) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f26ed681-a779-4b1f-be26-cb2487eb207e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline(steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65dfc297-c5ea-404a-a07f-893bb1230406",
   "metadata": {},
   "source": [
    "## 5.2) test du pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "266a89de-3c09-4f2d-8cd8-c5fd21e755f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 16)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indexes           = X_validation.sample(1).index\n",
    "df[target]        = df[target].map({\"no\":0, \"yes\":1})\n",
    "data_sans_dummies = df.iloc[indexes].drop(target, axis=1)\n",
    "y_a_tester        = df.iloc[indexes][target]\n",
    "data_sans_dummies.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e94fc4a5-d41a-423d-9d65-314f2d0070bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.99998012e-01, 1.98797207e-06]])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.predict_proba(data_sans_dummies)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e8ed1fa-7171-4c07-aada-0ad59fca9bdd",
   "metadata": {},
   "source": [
    "### 5.2.1) score de la prédiction globale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "72a07a75-8c68-4010-8acf-a7e01b211e98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2260, 16)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indexes           = X_validation.index\n",
    "data_sans_dummies = df.iloc[indexes].drop(target, axis=1)\n",
    "y_a_tester        = df.iloc[indexes][target]\n",
    "data_sans_dummies.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "030e090f-ea10-43c0-a082-10e4e5977d40",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_ = pipeline.predict(data_sans_dummies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "3bf9b1e4-94e3-4838-8857-2e914a1ba6bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "reel_ = y_a_tester.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8e73672d-b4d4-45f0-8dc2-9c6ad8499fca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9004424778761062"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.score(data_sans_dummies, y_a_tester.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65964ace-b2e6-42e3-a743-59c3a2b9dfd0",
   "metadata": {},
   "source": [
    "# 6) Save the pipeline model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "194a1134-aedc-4363-a8ad-0d56d0533cc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "652b8b2f-4f8f-4107-bc66-6f07d2e58f11",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_filename = \"model.pkl\"\n",
    "with open(model_filename, \"wb\") as f:\n",
    "    pickle.dump(pipeline, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4c307e2-9a3e-49a8-82ff-b99a8c8ead3f",
   "metadata": {},
   "source": [
    "# 7) Passer en production\n",
    "In order to deploy your trained model on AI Platform Prediction, you must:\n",
    "- Upload your saved model to a Cloud Storage bucket.\n",
    "- Create an AI Platform Prediction model resource.\n",
    "- Create an AI Platform Prediction version resource, specifying the Cloud Storage path to your saved model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "e9fa62ff-5806-4f5e-ae11-ed0e6e2eb76f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "4657f067-2f30-45c4-a3e5-97f5d5424af3",
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_NAME   = \"from_notebook_cloud_orbit\"\n",
    "VERSION_NAME = f\"{MODEL_NAME}_{int(time())}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "f177a66b-98f1-46e2-8797-a987407f043d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'from_notebook_cloud_orbit_1655758904'"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VERSION_NAME"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df4073f8-f5d1-42e7-8a8b-917e59c535c8",
   "metadata": {},
   "source": [
    "## 7.1) Upload the model to Cloud Storage => https://pantheon.corp.google.com/storage "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "d8bba7e2-ae51-47ee-b15a-0cad6b2967e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "BUCKET_NAME = \"bucket_cloud_orbit\"\n",
    "bucket      = client.bucket(BUCKET_NAME)\n",
    "blob        = bucket.blob(model_filename)\n",
    "blob.upload_from_filename(model_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "35985d90-5e60-429b-849f-6b19a3af5a0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Blob: bucket_cloud_orbit, 2020.csv, 1655678817572147>,\n",
       " <Blob: bucket_cloud_orbit, 2020_avec_virgules.csv, 1655679846234531>,\n",
       " <Blob: bucket_cloud_orbit, 2020_avec_virgules_one_line.csv, 1655680596719636>,\n",
       " <Blob: bucket_cloud_orbit, avro_schema.avsc, 1655720004709811>,\n",
       " <Blob: bucket_cloud_orbit, forecast.avro, 1655720031931821>,\n",
       " <Blob: bucket_cloud_orbit, model.pkl, 1655758905159882>,\n",
       " <Blob: bucket_cloud_orbit, test.pkl, 1655758895665852>,\n",
       " <Blob: bucket_cloud_orbit, une_col_une_ligne.csv, 1655681019154689>]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x for x in bucket.list_blobs()]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99ff469d-0718-4be5-a3e2-309bcbd3b5a2",
   "metadata": {},
   "source": [
    "## 7.2) Créer un endpoint, dans l'AI Plateforme (en cours d'intégration à Vertex AI ?) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5d2ad4d-5068-4d9a-8a34-6b5bbf26e14e",
   "metadata": {},
   "source": [
    "### 7.2.1) activer l'API Endpoint dans le projet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "4272a2ef-1d5e-4134-9ec2-ce20afba8525",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!gcloud services enable ml.googleapis.com"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ede9197-e623-4f2b-a833-1538c19d1ec1",
   "metadata": {},
   "source": [
    "### 7.2.2) Créer le modèle dans l'endpoint, Redirige vers l'AI Platform => https://pantheon.corp.google.com/ai-platform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "c402eaf6-a67b-465b-9375-54c802b7d6a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parfois il faut passer par la ligne de commande :-/ notamment pour répondre aux questions posées par les commandes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "7f30dc39-63bb-4ed4-957d-e8e27b8fee24",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fait à la main : https://pantheon.corp.google.com/vertex-ai/models?project=jouin-romain-demos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "9f049bd0-3581-4c59-bdcc-45dbb29ba5f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gcloud ai-platform models create from_notebook_cloud_orbit\n"
     ]
    }
   ],
   "source": [
    "print(f\"gcloud ai-platform models create {MODEL_NAME}\") # Chose region 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "e0d51597-cac6-41b2-9a61-d95cdd13ed68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using endpoint [https://us-central1-ml.googleapis.com/]\n",
      "Created ai platform model [projects/helical-sanctum-353821/models/from_notebook_cloud_orbit].\n"
     ]
    }
   ],
   "source": [
    "!gcloud ai-platform models create $MODEL_NAME --region=\"us-central1\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c921d411-0319-4965-81a1-bcf6cb738f9d",
   "metadata": {},
   "source": [
    "## 7.3) Créer une ressource versionnée qui servira le modèle (prend du temps quand ça plante : y a pas de feedback au niveau du terminal de commande...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "3468b980-ae3c-40e0-af13-26a9c4ecb83b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The system a \"model.pkl\" file into the directory \"origin\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "6a2e64ab-1217-4893-9978-5da76ea4966f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gcloud ai-platform versions create from_notebook_cloud_orbit_1655758904   --model=from_notebook_cloud_orbit   --framework=scikit-learn   --origin=gs://bucket_cloud_orbit  --python-version=3.5   --runtime-version=1.12 \n"
     ]
    }
   ],
   "source": [
    "print(f\"\"\"gcloud ai-platform versions create {VERSION_NAME} \\\n",
    "  --model={MODEL_NAME} \\\n",
    "  --framework=scikit-learn \\\n",
    "  --origin=gs://{BUCKET_NAME}\\\n",
    "  --python-version=3.5 \\\n",
    "  --runtime-version=1.12 \"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "db68e77e-ffdb-4a55-bee3-773218dafa9b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using endpoint [https://us-central1-ml.googleapis.com/]\n",
      "Creating version (this might take a few minutes)......failed.                  \n",
      "\u001b[1;31mERROR:\u001b[0m (gcloud.ai-platform.versions.create) Timeout. To view model server logs, please enable console logging when creating your model (https://cloud.google.com/ai-platform/prediction/docs/online-predict#requesting_logs_for_online_prediction_requests). If this error persists, contact cloudml-feedback@google.com\n"
     ]
    }
   ],
   "source": [
    "!gcloud ai-platform versions create $VERSION_NAME \\\n",
    "  --model=$MODEL_NAME \\\n",
    "  --framework=scikit-learn \\\n",
    "  --origin=gs://$BUCKET_NAME/ \\\n",
    "  --python-version=3.5 \\\n",
    "  --runtime-version=1.12 \\\n",
    "  --region=us-central1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bed41f6-a151-445f-80da-11925c33521d",
   "metadata": {},
   "source": [
    "# 8) testing the API "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "id": "52bb34b2-b616-424f-9a55-a51443c434fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Job</th>\n",
       "      <th>MaritalStatus</th>\n",
       "      <th>Education</th>\n",
       "      <th>Default</th>\n",
       "      <th>Balance</th>\n",
       "      <th>Housing</th>\n",
       "      <th>Loan</th>\n",
       "      <th>Contact</th>\n",
       "      <th>Day</th>\n",
       "      <th>Month</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Campaign</th>\n",
       "      <th>PDays</th>\n",
       "      <th>Previous</th>\n",
       "      <th>POutcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23801</th>\n",
       "      <td>36</td>\n",
       "      <td>technician</td>\n",
       "      <td>single</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>29</td>\n",
       "      <td>aug</td>\n",
       "      <td>85</td>\n",
       "      <td>6</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21209</th>\n",
       "      <td>45</td>\n",
       "      <td>technician</td>\n",
       "      <td>single</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>18</td>\n",
       "      <td>aug</td>\n",
       "      <td>773</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31422</th>\n",
       "      <td>57</td>\n",
       "      <td>management</td>\n",
       "      <td>divorced</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>27</td>\n",
       "      <td>mar</td>\n",
       "      <td>102</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4951</th>\n",
       "      <td>34</td>\n",
       "      <td>management</td>\n",
       "      <td>single</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>no</td>\n",
       "      <td>1350</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>21</td>\n",
       "      <td>may</td>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39400</th>\n",
       "      <td>39</td>\n",
       "      <td>management</td>\n",
       "      <td>single</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>22</td>\n",
       "      <td>may</td>\n",
       "      <td>271</td>\n",
       "      <td>1</td>\n",
       "      <td>93</td>\n",
       "      <td>3</td>\n",
       "      <td>success</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4440</th>\n",
       "      <td>36</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>465</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>20</td>\n",
       "      <td>may</td>\n",
       "      <td>160</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14750</th>\n",
       "      <td>53</td>\n",
       "      <td>blue-collar</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>6258</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>telephone</td>\n",
       "      <td>15</td>\n",
       "      <td>jul</td>\n",
       "      <td>62</td>\n",
       "      <td>5</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2375</th>\n",
       "      <td>32</td>\n",
       "      <td>management</td>\n",
       "      <td>single</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>yes</td>\n",
       "      <td>-3</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>13</td>\n",
       "      <td>may</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7700</th>\n",
       "      <td>41</td>\n",
       "      <td>blue-collar</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>629</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>unknown</td>\n",
       "      <td>30</td>\n",
       "      <td>may</td>\n",
       "      <td>208</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36599</th>\n",
       "      <td>37</td>\n",
       "      <td>blue-collar</td>\n",
       "      <td>married</td>\n",
       "      <td>primary</td>\n",
       "      <td>no</td>\n",
       "      <td>-106</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>cellular</td>\n",
       "      <td>12</td>\n",
       "      <td>may</td>\n",
       "      <td>690</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2260 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age          Job MaritalStatus  Education Default  Balance Housing  \\\n",
       "23801   36   technician        single  secondary      no        0      no   \n",
       "21209   45   technician        single   tertiary      no        0      no   \n",
       "31422   57   management      divorced    unknown      no        0      no   \n",
       "4951    34   management        single   tertiary      no     1350     yes   \n",
       "39400   39   management        single   tertiary      no        0     yes   \n",
       "...    ...          ...           ...        ...     ...      ...     ...   \n",
       "4440    36       admin.       married  secondary      no      465     yes   \n",
       "14750   53  blue-collar       married  secondary      no     6258     yes   \n",
       "2375    32   management        single   tertiary     yes       -3     yes   \n",
       "7700    41  blue-collar       married  secondary      no      629     yes   \n",
       "36599   37  blue-collar       married    primary      no     -106     yes   \n",
       "\n",
       "      Loan    Contact  Day Month  Duration  Campaign  PDays  Previous POutcome  \n",
       "23801   no   cellular   29   aug        85         6     -1         0  unknown  \n",
       "21209   no   cellular   18   aug       773         4     -1         0  unknown  \n",
       "31422   no   cellular   27   mar       102         1     -1         0  unknown  \n",
       "4951    no    unknown   21   may        68         1     -1         0  unknown  \n",
       "39400   no   cellular   22   may       271         1     93         3  success  \n",
       "...    ...        ...  ...   ...       ...       ...    ...       ...      ...  \n",
       "4440    no    unknown   20   may       160         1     -1         0  unknown  \n",
       "14750   no  telephone   15   jul        62         5     -1         0  unknown  \n",
       "2375    no    unknown   13   may       500         2     -1         0  unknown  \n",
       "7700   yes    unknown   30   may       208         2     -1         0  unknown  \n",
       "36599  yes   cellular   12   may       690         1     -1         0  unknown  \n",
       "\n",
       "[2260 rows x 16 columns]"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sans_dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "8abb2f80-365e-4243-aa50-c042f0a65866",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "data = data_sans_dummies[0:1].to_json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "463bf1fc-f359-4d29-ba7b-b7cb627b815b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dico = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "3d06838a-858a-4955-9621-a22195ed57d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dico[\"instances\"] = [data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "63d3d95d-0af6-4227-8237-aff1e30c1078",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "f93239fa-5d87-4eff-9c22-1b726a7523d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data.json\", \"w\") as f: \n",
    "    json.dump(dico, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "d623b60c-e6b3-4601-a8c6-a19a04496b02",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (1571311334.py, line 2)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"/tmp/ipykernel_17340/1571311334.py\"\u001b[0;36m, line \u001b[0;32m2\u001b[0m\n\u001b[0;31m    -X POST \\\u001b[0m\n\u001b[0m          ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "curl \\\n",
    "-X POST \\\n",
    "-H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
    "-H \"Content-Type: application/json\" \\\n",
    "https://europe-west1-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/europe-west1/endpoints/${ENDPOINT_ID}:predict \\\n",
    "-d \"@${data.json}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "55a61ced-9a43-4b8a-88a6-22277b6bace7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Age': {12374: 57},\n",
       " 'Job': {12374: 'housemaid'},\n",
       " 'MaritalStatus': {12374: 'married'},\n",
       " 'Education': {12374: 'primary'},\n",
       " 'Default': {12374: 'no'},\n",
       " 'Balance': {12374: 85},\n",
       " 'Housing': {12374: 'no'},\n",
       " 'Loan': {12374: 'no'},\n",
       " 'Contact': {12374: 'unknown'},\n",
       " 'Day': {12374: 27},\n",
       " 'Month': {12374: 'jun'},\n",
       " 'Duration': {12374: 194},\n",
       " 'Campaign': {12374: 1},\n",
       " 'PDays': {12374: -1},\n",
       " 'Previous': {12374: 0},\n",
       " 'POutcome': {12374: 'unknown'}}"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sans_dummies[0:1].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4d405d5-05c4-47a7-bbcf-99068ade05b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "{\"instances\" : [[57, 'housemaid', 'married', 'primary', 'no', 85, 'no', 'no',\n",
    "        'unknown', 27, 'jun', 194, 1, -1, 0, 'unknown', '?']] }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "e93b73de-4f58-4fdc-a89b-008ec894820c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"a\" : [57],\n",
      "\"z\" : [\"housemaid\"],\n",
      "\"e\" : [\"married\"],\n",
      "\"r\" : [\"primary\"],\n",
      "\"t\" : [\"no\"],\n",
      "\"\"y : [ 85],\n",
      "\"u\" : [\"no\"],\n",
      "\"i\" : [\"no\"],\n",
      "\"o\" : [\"unknown\"],\n",
      "\"p\" : [ 27],\n",
      "\"m\" : [\"jun\"],\n",
      "\"k\" : [ 194],\n",
      "\"j : [ 1],\n",
      "\"f\" : [-1],\n",
      "\"h\" : [ 0],\n",
      "\"g\" : [\"unknown\"]}\n"
     ]
    }
   ],
   "source": [
    "!cat data.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "b42d598a-3634-4c13-8d4f-f21827469c16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"Age\":{\"12374\":57},\"Job\":{\"12374\":\"housemaid\"},\"MaritalStatus\":{\"12374\":\"married\"},\"Education\":{\"12374\":\"primary\"},\"Default\":{\"12374\":\"no\"},\"Balance\":{\"12374\":85},\"Housing\":{\"12374\":\"no\"},\"Loan\":{\"12374\":\"no\"},\"Contact\":{\"12374\":\"unknown\"},\"Day\":{\"12374\":27},\"Month\":{\"12374\":\"jun\"},\"Duration\":{\"12374\":194},\"Campaign\":{\"12374\":1},\"PDays\":{\"12374\":-1},\"Previous\":{\"12374\":0},\"POutcome\":{\"12374\":\"unknown\"}}'"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb15c8d1-f6d0-4d48-8efc-533fbc5a9da7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "common-cpu.m109",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m109"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}