03_model_quality_assurance_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a70cd9ae",
   "metadata": {},
   "source": [
    "# Prepare data for model quality assurance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "04ab4833",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pathlib\n",
    "import numpy as np\n",
    "from assets.hierarchy import get_hierarchy_lookup\n",
    "from assets.functions import gate_for_patient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cf13e400",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data\n",
    "data_model = pd.read_pickle(\n",
    "    pathlib.Path(r'data\\\\processed_data\\\\#1_aicontrol_dataset_pred.pkl')\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c59a2b93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert multiindex into columns\n",
    "data_model.reset_index(inplace=True, drop=False, col_level=1)\n",
    "# drop header to easy data handling\n",
    "data_model = data_model.droplevel(axis=1, level=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5d4fe772",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>event</th>\n",
       "      <th>FSC-A</th>\n",
       "      <th>SSC-A</th>\n",
       "      <th>FITC-A</th>\n",
       "      <th>PE-A</th>\n",
       "      <th>PerCP-A</th>\n",
       "      <th>PE-Cy7-A</th>\n",
       "      <th>APC-A</th>\n",
       "      <th>APC-H7-A</th>\n",
       "      <th>...</th>\n",
       "      <th>TBNK Sum</th>\n",
       "      <th>TP</th>\n",
       "      <th>valid_Q_file</th>\n",
       "      <th>used_for_training_Q_file</th>\n",
       "      <th>pred_Lympho</th>\n",
       "      <th>pred_BP</th>\n",
       "      <th>pred_NKP</th>\n",
       "      <th>pred_TP</th>\n",
       "      <th>pred_T4P</th>\n",
       "      <th>pred_T8P</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>001</td>\n",
       "      <td>0</td>\n",
       "      <td>50573.101562</td>\n",
       "      <td>9443.280273</td>\n",
       "      <td>-192.361191</td>\n",
       "      <td>49.568016</td>\n",
       "      <td>127.291069</td>\n",
       "      <td>-25.279428</td>\n",
       "      <td>-207.001297</td>\n",
       "      <td>49765.726562</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>001</td>\n",
       "      <td>1</td>\n",
       "      <td>60721.648438</td>\n",
       "      <td>15401.190430</td>\n",
       "      <td>50677.765625</td>\n",
       "      <td>11363.137695</td>\n",
       "      <td>607.533691</td>\n",
       "      <td>947.504639</td>\n",
       "      <td>-62.260956</td>\n",
       "      <td>21956.218750</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001</td>\n",
       "      <td>2</td>\n",
       "      <td>38507.070312</td>\n",
       "      <td>11470.410156</td>\n",
       "      <td>102.436935</td>\n",
       "      <td>13033.757812</td>\n",
       "      <td>683.279968</td>\n",
       "      <td>4808.329102</td>\n",
       "      <td>-151.480698</td>\n",
       "      <td>41288.160156</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001</td>\n",
       "      <td>3</td>\n",
       "      <td>88148.398438</td>\n",
       "      <td>88019.195312</td>\n",
       "      <td>99.228241</td>\n",
       "      <td>836.311340</td>\n",
       "      <td>-139.178345</td>\n",
       "      <td>2764.074951</td>\n",
       "      <td>67.033386</td>\n",
       "      <td>11105.057617</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001</td>\n",
       "      <td>4</td>\n",
       "      <td>71808.062500</td>\n",
       "      <td>86550.664062</td>\n",
       "      <td>128.604401</td>\n",
       "      <td>1799.095459</td>\n",
       "      <td>123.587227</td>\n",
       "      <td>5954.442383</td>\n",
       "      <td>97.762772</td>\n",
       "      <td>14570.107422</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6887771</th>\n",
       "      <td>100</td>\n",
       "      <td>76983</td>\n",
       "      <td>73832.546875</td>\n",
       "      <td>74114.460938</td>\n",
       "      <td>163.966949</td>\n",
       "      <td>1048.523315</td>\n",
       "      <td>309.503815</td>\n",
       "      <td>10503.620117</td>\n",
       "      <td>473.652069</td>\n",
       "      <td>11203.947266</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6887772</th>\n",
       "      <td>100</td>\n",
       "      <td>76984</td>\n",
       "      <td>56489.968750</td>\n",
       "      <td>44520.421875</td>\n",
       "      <td>61.773682</td>\n",
       "      <td>560.338135</td>\n",
       "      <td>-157.014450</td>\n",
       "      <td>6751.536133</td>\n",
       "      <td>240.275589</td>\n",
       "      <td>8255.482422</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6887773</th>\n",
       "      <td>100</td>\n",
       "      <td>76985</td>\n",
       "      <td>63175.050781</td>\n",
       "      <td>67636.171875</td>\n",
       "      <td>79.222885</td>\n",
       "      <td>752.837341</td>\n",
       "      <td>251.232635</td>\n",
       "      <td>12247.950195</td>\n",
       "      <td>54.703140</td>\n",
       "      <td>10784.350586</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6887774</th>\n",
       "      <td>100</td>\n",
       "      <td>76986</td>\n",
       "      <td>56315.968750</td>\n",
       "      <td>24900.330078</td>\n",
       "      <td>5775.441406</td>\n",
       "      <td>8930.164062</td>\n",
       "      <td>-378.949402</td>\n",
       "      <td>1057.262939</td>\n",
       "      <td>382.759033</td>\n",
       "      <td>29675.646484</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6887775</th>\n",
       "      <td>100</td>\n",
       "      <td>76987</td>\n",
       "      <td>74095.289062</td>\n",
       "      <td>37668.750000</td>\n",
       "      <td>2597.294189</td>\n",
       "      <td>7626.439941</td>\n",
       "      <td>-46.928322</td>\n",
       "      <td>201.112900</td>\n",
       "      <td>58.847775</td>\n",
       "      <td>27282.300781</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6887776 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        file  event         FSC-A         SSC-A        FITC-A          PE-A  \\\n",
       "0        001      0  50573.101562   9443.280273   -192.361191     49.568016   \n",
       "1        001      1  60721.648438  15401.190430  50677.765625  11363.137695   \n",
       "2        001      2  38507.070312  11470.410156    102.436935  13033.757812   \n",
       "3        001      3  88148.398438  88019.195312     99.228241    836.311340   \n",
       "4        001      4  71808.062500  86550.664062    128.604401   1799.095459   \n",
       "...      ...    ...           ...           ...           ...           ...   \n",
       "6887771  100  76983  73832.546875  74114.460938    163.966949   1048.523315   \n",
       "6887772  100  76984  56489.968750  44520.421875     61.773682    560.338135   \n",
       "6887773  100  76985  63175.050781  67636.171875     79.222885    752.837341   \n",
       "6887774  100  76986  56315.968750  24900.330078   5775.441406   8930.164062   \n",
       "6887775  100  76987  74095.289062  37668.750000   2597.294189   7626.439941   \n",
       "\n",
       "            PerCP-A      PE-Cy7-A       APC-A      APC-H7-A  ...  TBNK Sum  \\\n",
       "0        127.291069    -25.279428 -207.001297  49765.726562  ...         1   \n",
       "1        607.533691    947.504639  -62.260956  21956.218750  ...         0   \n",
       "2        683.279968   4808.329102 -151.480698  41288.160156  ...         1   \n",
       "3       -139.178345   2764.074951   67.033386  11105.057617  ...         0   \n",
       "4        123.587227   5954.442383   97.762772  14570.107422  ...         0   \n",
       "...             ...           ...         ...           ...  ...       ...   \n",
       "6887771  309.503815  10503.620117  473.652069  11203.947266  ...         0   \n",
       "6887772 -157.014450   6751.536133  240.275589   8255.482422  ...         0   \n",
       "6887773  251.232635  12247.950195   54.703140  10784.350586  ...         0   \n",
       "6887774 -378.949402   1057.262939  382.759033  29675.646484  ...         0   \n",
       "6887775  -46.928322    201.112900   58.847775  27282.300781  ...         0   \n",
       "\n",
       "         TP  valid_Q_file  used_for_training_Q_file  pred_Lympho  pred_BP  \\\n",
       "0         1             1                         0            1        0   \n",
       "1         0             1                         0            0        0   \n",
       "2         0             1                         0            1        0   \n",
       "3         0             1                         0            0        0   \n",
       "4         0             1                         0            0        0   \n",
       "...      ..           ...                       ...          ...      ...   \n",
       "6887771   0             1                         0            0        0   \n",
       "6887772   0             1                         0            0        0   \n",
       "6887773   0             1                         0            0        0   \n",
       "6887774   0             1                         0            0        0   \n",
       "6887775   0             1                         0            0        0   \n",
       "\n",
       "         pred_NKP  pred_TP  pred_T4P  pred_T8P  \n",
       "0               0        1         0         0  \n",
       "1               0        0         0         0  \n",
       "2               1        0         0         0  \n",
       "3               0        0         0         0  \n",
       "4               0        0         0         0  \n",
       "...           ...      ...       ...       ...  \n",
       "6887771         0        0         0         0  \n",
       "6887772         0        0         0         0  \n",
       "6887773         0        0         0         0  \n",
       "6887774         0        0         0         0  \n",
       "6887775         0        0         0         0  \n",
       "\n",
       "[6887776 rows x 67 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ba694717",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get traning data\n",
    "data_train = data_model.loc[data_model['used_for_training_Q_file'] == 1]\n",
    "# Get patient IDs of events used for training -> should not be used for quality assurance\n",
    "patients_train = data_train.loc[:, 'file'].unique()\n",
    "# get test data\n",
    "data_test = data_model[~data_model['file'].isin(patients_train)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ca30ca22",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save training and test datasets\n",
    "data_train.to_pickle(pathlib.Path(r'data/processed_data/dataset_pred_train.pkl'))\n",
    "data_test.to_pickle(pathlib.Path(r'data/processed_data/dataset_pred_test.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4e1bac85",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save complete dataset (reformatted)\n",
    "data_model.to_pickle(pathlib.Path(r'data/processed_data/dataset_pred_complete.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "04f4eb52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save patient IDs of training events\n",
    "pd.Series(patients_train).to_pickle(pathlib.Path(r'data/processed_data/patient_ids_train.pkl'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3aa5e7e9",
   "metadata": {},
   "source": [
    "## Sum events per sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b0468a32",
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of celltypes considered by the model\n",
    "celltypes_ddm = ['Lympho', 'BP','NKP', 'TP', 'T4P', 'T8P']\n",
    "\n",
    "# list of column names for predicted events by the model\n",
    "celltypes_pred = [f'pred_{celltype}' for celltype in celltypes_ddm]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4e5efee1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# names of columns that will be summed\n",
    "sum_cols = celltypes_ddm + celltypes_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "966982b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lympho</th>\n",
       "      <th>BP</th>\n",
       "      <th>NKP</th>\n",
       "      <th>TP</th>\n",
       "      <th>T4P</th>\n",
       "      <th>T8P</th>\n",
       "      <th>pred_Lympho</th>\n",
       "      <th>pred_BP</th>\n",
       "      <th>pred_NKP</th>\n",
       "      <th>pred_TP</th>\n",
       "      <th>pred_T4P</th>\n",
       "      <th>pred_T8P</th>\n",
       "      <th>total</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>file</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>001</th>\n",
       "      <td>18566</td>\n",
       "      <td>2798</td>\n",
       "      <td>6567</td>\n",
       "      <td>9242</td>\n",
       "      <td>5719</td>\n",
       "      <td>2919</td>\n",
       "      <td>18537</td>\n",
       "      <td>2788</td>\n",
       "      <td>6060</td>\n",
       "      <td>9219</td>\n",
       "      <td>5684</td>\n",
       "      <td>2919</td>\n",
       "      <td>49450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>002</th>\n",
       "      <td>18647</td>\n",
       "      <td>2620</td>\n",
       "      <td>3062</td>\n",
       "      <td>12725</td>\n",
       "      <td>8924</td>\n",
       "      <td>2656</td>\n",
       "      <td>18475</td>\n",
       "      <td>2597</td>\n",
       "      <td>3015</td>\n",
       "      <td>12714</td>\n",
       "      <td>8921</td>\n",
       "      <td>2734</td>\n",
       "      <td>61105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>003</th>\n",
       "      <td>19040</td>\n",
       "      <td>1849</td>\n",
       "      <td>1665</td>\n",
       "      <td>15422</td>\n",
       "      <td>7985</td>\n",
       "      <td>5678</td>\n",
       "      <td>18992</td>\n",
       "      <td>1834</td>\n",
       "      <td>1618</td>\n",
       "      <td>15404</td>\n",
       "      <td>7969</td>\n",
       "      <td>5670</td>\n",
       "      <td>74653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>004</th>\n",
       "      <td>19217</td>\n",
       "      <td>2465</td>\n",
       "      <td>3792</td>\n",
       "      <td>12789</td>\n",
       "      <td>7466</td>\n",
       "      <td>4244</td>\n",
       "      <td>19267</td>\n",
       "      <td>2470</td>\n",
       "      <td>3781</td>\n",
       "      <td>12798</td>\n",
       "      <td>7468</td>\n",
       "      <td>4246</td>\n",
       "      <td>60175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>005</th>\n",
       "      <td>17963</td>\n",
       "      <td>1607</td>\n",
       "      <td>2376</td>\n",
       "      <td>14355</td>\n",
       "      <td>10509</td>\n",
       "      <td>3241</td>\n",
       "      <td>18099</td>\n",
       "      <td>1618</td>\n",
       "      <td>1940</td>\n",
       "      <td>14383</td>\n",
       "      <td>10478</td>\n",
       "      <td>3248</td>\n",
       "      <td>97513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>096</th>\n",
       "      <td>17477</td>\n",
       "      <td>3483</td>\n",
       "      <td>882</td>\n",
       "      <td>12775</td>\n",
       "      <td>8996</td>\n",
       "      <td>3058</td>\n",
       "      <td>17273</td>\n",
       "      <td>3337</td>\n",
       "      <td>824</td>\n",
       "      <td>12718</td>\n",
       "      <td>8823</td>\n",
       "      <td>3037</td>\n",
       "      <td>87710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>097</th>\n",
       "      <td>19276</td>\n",
       "      <td>1495</td>\n",
       "      <td>2627</td>\n",
       "      <td>15041</td>\n",
       "      <td>8235</td>\n",
       "      <td>5146</td>\n",
       "      <td>18684</td>\n",
       "      <td>1394</td>\n",
       "      <td>2156</td>\n",
       "      <td>14916</td>\n",
       "      <td>8034</td>\n",
       "      <td>5129</td>\n",
       "      <td>56295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>098</th>\n",
       "      <td>18543</td>\n",
       "      <td>2220</td>\n",
       "      <td>3489</td>\n",
       "      <td>12619</td>\n",
       "      <td>8667</td>\n",
       "      <td>3243</td>\n",
       "      <td>18195</td>\n",
       "      <td>2145</td>\n",
       "      <td>3300</td>\n",
       "      <td>12529</td>\n",
       "      <td>8500</td>\n",
       "      <td>3237</td>\n",
       "      <td>61959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>099</th>\n",
       "      <td>17494</td>\n",
       "      <td>2102</td>\n",
       "      <td>5213</td>\n",
       "      <td>9880</td>\n",
       "      <td>5658</td>\n",
       "      <td>3771</td>\n",
       "      <td>17316</td>\n",
       "      <td>2039</td>\n",
       "      <td>5072</td>\n",
       "      <td>9869</td>\n",
       "      <td>5607</td>\n",
       "      <td>3769</td>\n",
       "      <td>50206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>17470</td>\n",
       "      <td>1126</td>\n",
       "      <td>4068</td>\n",
       "      <td>12148</td>\n",
       "      <td>7249</td>\n",
       "      <td>3153</td>\n",
       "      <td>17337</td>\n",
       "      <td>1094</td>\n",
       "      <td>3880</td>\n",
       "      <td>12126</td>\n",
       "      <td>7165</td>\n",
       "      <td>3197</td>\n",
       "      <td>76988</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Lympho    BP   NKP     TP    T4P   T8P  pred_Lympho  pred_BP  pred_NKP  \\\n",
       "file                                                                           \n",
       "001    18566  2798  6567   9242   5719  2919        18537     2788      6060   \n",
       "002    18647  2620  3062  12725   8924  2656        18475     2597      3015   \n",
       "003    19040  1849  1665  15422   7985  5678        18992     1834      1618   \n",
       "004    19217  2465  3792  12789   7466  4244        19267     2470      3781   \n",
       "005    17963  1607  2376  14355  10509  3241        18099     1618      1940   \n",
       "...      ...   ...   ...    ...    ...   ...          ...      ...       ...   \n",
       "096    17477  3483   882  12775   8996  3058        17273     3337       824   \n",
       "097    19276  1495  2627  15041   8235  5146        18684     1394      2156   \n",
       "098    18543  2220  3489  12619   8667  3243        18195     2145      3300   \n",
       "099    17494  2102  5213   9880   5658  3771        17316     2039      5072   \n",
       "100    17470  1126  4068  12148   7249  3153        17337     1094      3880   \n",
       "\n",
       "      pred_TP  pred_T4P  pred_T8P  total  \n",
       "file                                      \n",
       "001      9219      5684      2919  49450  \n",
       "002     12714      8921      2734  61105  \n",
       "003     15404      7969      5670  74653  \n",
       "004     12798      7468      4246  60175  \n",
       "005     14383     10478      3248  97513  \n",
       "...       ...       ...       ...    ...  \n",
       "096     12718      8823      3037  87710  \n",
       "097     14916      8034      5129  56295  \n",
       "098     12529      8500      3237  61959  \n",
       "099      9869      5607      3769  50206  \n",
       "100     12126      7165      3197  76988  \n",
       "\n",
       "[100 rows x 13 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "event_sum_per_sample = data_model.loc[:, ['file'] + sum_cols].groupby('file').sum()\n",
    "\n",
    "event_sum_per_sample.loc[:, 'total'] = data_model.loc[:, ['file'] + sum_cols].groupby('file').size()\n",
    "\n",
    "event_sum_per_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "254ecd85",
   "metadata": {},
   "outputs": [],
   "source": [
    "event_sum_per_sample.to_pickle(pathlib.Path(r'data/processed_data/event_sum_per_sample_complete.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b2c03a2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "celltypes_pred.append('total')\n",
    "event_sum_per_sample[celltypes_pred].to_pickle(pathlib.Path(r'data/processed_data/model_summed.pkl'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec3f8449",
   "metadata": {},
   "source": [
    "## Determine manual linear gating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "db012d00",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Names of the marker features\n",
    "marker_names = ['FSC-A', 'SSC-A', 'APC-A', 'APC-H7-A', 'AmCyan-A', 'FITC-A', 'PE-A', \n",
    "                'PE-Cy7-A', 'Pacific Blue-A', 'PerCP-A']\n",
    "\n",
    "# Names of the ground truth features of all cell types\n",
    "gt_names_all = ['BP', 'CD4', 'CD4x38',\n",
    "       'CD4xDR', 'CD4xDRx38', 'CD8', 'CD8x38', 'CD8xDR', 'CD8xDRx38', 'Lympho',\n",
    "       'Manja rufen!', 'NKP', 'NKTP', 'NKx38', 'NKxDR', 'NKxDRx38', 'Q1-1',\n",
    "       'Q1-2', 'Q1-3', 'Q1-5', 'Q1-6', 'Q1-7', 'Q1-9', 'Q2', 'Q3', 'Q3-1',\n",
    "       'Q3-2', 'Q3-4', 'Q3-5', 'Q3-6', 'Q3-7', 'Q3-9', 'Q4-1', 'Q4-2', 'Q4-3',\n",
    "       'Q4-4', 'Q4-5', 'Q4-6', 'Q4-7', 'Q4-9', 'T48NT', 'T48PT', 'T4P', 'T8P',\n",
    "       'TBNK Sum', 'TP']\n",
    "\n",
    "# Names of the ground truth features of selected cell types, i.e. cell types that are also predicted by the DDM\n",
    "gt_names_selected = ['Lympho', 'T4P', 'T8P', 'BP', 'TP', 'NKP']\n",
    "\n",
    "# Names of the features for the predicted cell types\n",
    "pred_names_selected = ['pred_{}'.format(celltype) for celltype in gt_names_selected]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4a4b4fd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierarchy_lookup = get_hierarchy_lookup()\n",
    "\n",
    "# Names of celltypes determined in gating plots\n",
    "gating_celltype_names = list(hierarchy_lookup.loc[hierarchy_lookup['marker_x'].notna()].index.values)\n",
    "gating_celltype_names.remove('Lympho')\n",
    "gating_celltype_names.remove('Lympho_alternative')\n",
    "\n",
    "gating_celltype_names.remove('NKxDRx38')\n",
    "gating_celltype_names.remove('CD4xDRx38')\n",
    "gating_celltype_names.remove('CD8xDRx38')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9767cf7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create column Not(TP)\n",
    "data_model.loc[:, 'Not_TP'] = (~data_model.loc[:, 'TP'].astype(bool)).astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4fbce39f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# determine the x and y axis position of linear gates\n",
    "gating_df = []\n",
    "\n",
    "for patient_id in data_model['file'].unique():\n",
    "    gating_df.append(gate_for_patient(patient_id, data_model))\n",
    "    \n",
    "gating_df = pd.concat(gating_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0aa872e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BP_AmCyan-A_q12</th>\n",
       "      <th>BP_AmCyan-A_q34</th>\n",
       "      <th>BP_APC-A_q13</th>\n",
       "      <th>BP_APC-A_q24</th>\n",
       "      <th>TP_AmCyan-A_q12</th>\n",
       "      <th>TP_AmCyan-A_q34</th>\n",
       "      <th>TP_APC-A_q13</th>\n",
       "      <th>TP_APC-A_q24</th>\n",
       "      <th>T48PT_Pacific Blue-A_q12</th>\n",
       "      <th>T48PT_Pacific Blue-A_q34</th>\n",
       "      <th>...</th>\n",
       "      <th>NKTP_PE-Cy7-A_q13</th>\n",
       "      <th>NKTP_PE-Cy7-A_q24</th>\n",
       "      <th>CD4_AmCyan-A_q12</th>\n",
       "      <th>CD4_AmCyan-A_q34</th>\n",
       "      <th>CD4_Pacific Blue-A_q13</th>\n",
       "      <th>CD4_Pacific Blue-A_q24</th>\n",
       "      <th>CD8_AmCyan-A_q12</th>\n",
       "      <th>CD8_AmCyan-A_q34</th>\n",
       "      <th>CD8_PerCP-A_q13</th>\n",
       "      <th>CD8_PerCP-A_q24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>001</th>\n",
       "      <td>461.408005</td>\n",
       "      <td>460.324051</td>\n",
       "      <td>1546.691833</td>\n",
       "      <td>1546.691833</td>\n",
       "      <td>461.408005</td>\n",
       "      <td>460.324051</td>\n",
       "      <td>1546.691833</td>\n",
       "      <td>1546.691833</td>\n",
       "      <td>233.502617</td>\n",
       "      <td>233.502617</td>\n",
       "      <td>...</td>\n",
       "      <td>1073.911011</td>\n",
       "      <td>1078.053223</td>\n",
       "      <td>487.012085</td>\n",
       "      <td>498.808182</td>\n",
       "      <td>211.248085</td>\n",
       "      <td>211.248085</td>\n",
       "      <td>504.310776</td>\n",
       "      <td>487.861938</td>\n",
       "      <td>990.332001</td>\n",
       "      <td>3825.442261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>002</th>\n",
       "      <td>465.573700</td>\n",
       "      <td>461.550323</td>\n",
       "      <td>1549.481140</td>\n",
       "      <td>1549.481140</td>\n",
       "      <td>465.573700</td>\n",
       "      <td>461.550323</td>\n",
       "      <td>1549.481140</td>\n",
       "      <td>1549.481140</td>\n",
       "      <td>186.389404</td>\n",
       "      <td>186.389404</td>\n",
       "      <td>...</td>\n",
       "      <td>11729.672852</td>\n",
       "      <td>1558.832581</td>\n",
       "      <td>590.344879</td>\n",
       "      <td>594.977478</td>\n",
       "      <td>312.575684</td>\n",
       "      <td>312.575684</td>\n",
       "      <td>490.223831</td>\n",
       "      <td>490.223831</td>\n",
       "      <td>1201.658386</td>\n",
       "      <td>6921.157471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>003</th>\n",
       "      <td>451.195740</td>\n",
       "      <td>451.195740</td>\n",
       "      <td>1802.897827</td>\n",
       "      <td>1802.897827</td>\n",
       "      <td>451.195740</td>\n",
       "      <td>451.195740</td>\n",
       "      <td>1802.897827</td>\n",
       "      <td>1802.897827</td>\n",
       "      <td>200.654549</td>\n",
       "      <td>200.654549</td>\n",
       "      <td>...</td>\n",
       "      <td>2032.824158</td>\n",
       "      <td>760.268158</td>\n",
       "      <td>587.791046</td>\n",
       "      <td>591.941376</td>\n",
       "      <td>303.369293</td>\n",
       "      <td>303.369293</td>\n",
       "      <td>471.380859</td>\n",
       "      <td>479.999893</td>\n",
       "      <td>1184.574951</td>\n",
       "      <td>3082.143188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>004</th>\n",
       "      <td>462.813980</td>\n",
       "      <td>462.813980</td>\n",
       "      <td>1613.322815</td>\n",
       "      <td>1613.322815</td>\n",
       "      <td>462.813980</td>\n",
       "      <td>462.813980</td>\n",
       "      <td>1613.322815</td>\n",
       "      <td>1613.322815</td>\n",
       "      <td>212.288345</td>\n",
       "      <td>212.288345</td>\n",
       "      <td>...</td>\n",
       "      <td>2025.792664</td>\n",
       "      <td>1355.437561</td>\n",
       "      <td>596.772156</td>\n",
       "      <td>596.772156</td>\n",
       "      <td>212.790939</td>\n",
       "      <td>212.790939</td>\n",
       "      <td>484.688721</td>\n",
       "      <td>492.254013</td>\n",
       "      <td>1183.739258</td>\n",
       "      <td>2999.500122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>005</th>\n",
       "      <td>462.004196</td>\n",
       "      <td>462.004196</td>\n",
       "      <td>1568.070068</td>\n",
       "      <td>1568.070068</td>\n",
       "      <td>462.004196</td>\n",
       "      <td>462.004196</td>\n",
       "      <td>1568.070068</td>\n",
       "      <td>1568.070068</td>\n",
       "      <td>195.807983</td>\n",
       "      <td>195.807983</td>\n",
       "      <td>...</td>\n",
       "      <td>2566.633301</td>\n",
       "      <td>1253.903687</td>\n",
       "      <td>402.203644</td>\n",
       "      <td>402.203644</td>\n",
       "      <td>206.456284</td>\n",
       "      <td>206.456284</td>\n",
       "      <td>482.189163</td>\n",
       "      <td>487.244675</td>\n",
       "      <td>1170.567078</td>\n",
       "      <td>3372.401001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>096</th>\n",
       "      <td>468.626633</td>\n",
       "      <td>467.077530</td>\n",
       "      <td>1980.721741</td>\n",
       "      <td>1980.721741</td>\n",
       "      <td>468.626633</td>\n",
       "      <td>467.077530</td>\n",
       "      <td>1980.721741</td>\n",
       "      <td>1980.721741</td>\n",
       "      <td>213.936401</td>\n",
       "      <td>213.936401</td>\n",
       "      <td>...</td>\n",
       "      <td>2906.513062</td>\n",
       "      <td>1249.622498</td>\n",
       "      <td>460.437057</td>\n",
       "      <td>457.869614</td>\n",
       "      <td>200.150414</td>\n",
       "      <td>200.150414</td>\n",
       "      <td>427.757172</td>\n",
       "      <td>432.122986</td>\n",
       "      <td>1040.649231</td>\n",
       "      <td>3051.995850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>097</th>\n",
       "      <td>465.489029</td>\n",
       "      <td>465.489029</td>\n",
       "      <td>2349.541382</td>\n",
       "      <td>2349.541382</td>\n",
       "      <td>465.489029</td>\n",
       "      <td>465.489029</td>\n",
       "      <td>2349.541382</td>\n",
       "      <td>2349.541382</td>\n",
       "      <td>203.944809</td>\n",
       "      <td>203.944809</td>\n",
       "      <td>...</td>\n",
       "      <td>1305.977539</td>\n",
       "      <td>1294.955444</td>\n",
       "      <td>465.489029</td>\n",
       "      <td>465.489029</td>\n",
       "      <td>144.499069</td>\n",
       "      <td>144.499069</td>\n",
       "      <td>440.895248</td>\n",
       "      <td>440.895248</td>\n",
       "      <td>1013.062683</td>\n",
       "      <td>2920.687866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>098</th>\n",
       "      <td>462.685913</td>\n",
       "      <td>467.453308</td>\n",
       "      <td>1914.418884</td>\n",
       "      <td>1914.418884</td>\n",
       "      <td>462.685913</td>\n",
       "      <td>467.453308</td>\n",
       "      <td>1914.418884</td>\n",
       "      <td>1914.418884</td>\n",
       "      <td>232.355072</td>\n",
       "      <td>232.355072</td>\n",
       "      <td>...</td>\n",
       "      <td>3014.272339</td>\n",
       "      <td>1272.894592</td>\n",
       "      <td>477.395081</td>\n",
       "      <td>477.395081</td>\n",
       "      <td>155.943855</td>\n",
       "      <td>155.943855</td>\n",
       "      <td>371.646011</td>\n",
       "      <td>371.646011</td>\n",
       "      <td>825.394165</td>\n",
       "      <td>2151.710449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>099</th>\n",
       "      <td>472.083267</td>\n",
       "      <td>466.991913</td>\n",
       "      <td>2254.812256</td>\n",
       "      <td>2254.812256</td>\n",
       "      <td>472.083267</td>\n",
       "      <td>466.991913</td>\n",
       "      <td>2254.812256</td>\n",
       "      <td>2254.812256</td>\n",
       "      <td>203.208473</td>\n",
       "      <td>203.208473</td>\n",
       "      <td>...</td>\n",
       "      <td>3049.025513</td>\n",
       "      <td>1068.188843</td>\n",
       "      <td>481.194122</td>\n",
       "      <td>484.080368</td>\n",
       "      <td>169.896675</td>\n",
       "      <td>169.896675</td>\n",
       "      <td>423.866104</td>\n",
       "      <td>423.866104</td>\n",
       "      <td>932.293732</td>\n",
       "      <td>3027.066162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>360.690430</td>\n",
       "      <td>360.690430</td>\n",
       "      <td>1791.389404</td>\n",
       "      <td>1791.389404</td>\n",
       "      <td>360.690430</td>\n",
       "      <td>360.690430</td>\n",
       "      <td>1791.389404</td>\n",
       "      <td>1791.389404</td>\n",
       "      <td>203.381996</td>\n",
       "      <td>203.381996</td>\n",
       "      <td>...</td>\n",
       "      <td>3031.520630</td>\n",
       "      <td>907.434998</td>\n",
       "      <td>464.351974</td>\n",
       "      <td>464.351974</td>\n",
       "      <td>167.162354</td>\n",
       "      <td>167.162354</td>\n",
       "      <td>296.643661</td>\n",
       "      <td>296.643661</td>\n",
       "      <td>651.142914</td>\n",
       "      <td>2955.517822</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     BP_AmCyan-A_q12  BP_AmCyan-A_q34  BP_APC-A_q13  BP_APC-A_q24  \\\n",
       "001       461.408005       460.324051   1546.691833   1546.691833   \n",
       "002       465.573700       461.550323   1549.481140   1549.481140   \n",
       "003       451.195740       451.195740   1802.897827   1802.897827   \n",
       "004       462.813980       462.813980   1613.322815   1613.322815   \n",
       "005       462.004196       462.004196   1568.070068   1568.070068   \n",
       "..               ...              ...           ...           ...   \n",
       "096       468.626633       467.077530   1980.721741   1980.721741   \n",
       "097       465.489029       465.489029   2349.541382   2349.541382   \n",
       "098       462.685913       467.453308   1914.418884   1914.418884   \n",
       "099       472.083267       466.991913   2254.812256   2254.812256   \n",
       "100       360.690430       360.690430   1791.389404   1791.389404   \n",
       "\n",
       "     TP_AmCyan-A_q12  TP_AmCyan-A_q34  TP_APC-A_q13  TP_APC-A_q24  \\\n",
       "001       461.408005       460.324051   1546.691833   1546.691833   \n",
       "002       465.573700       461.550323   1549.481140   1549.481140   \n",
       "003       451.195740       451.195740   1802.897827   1802.897827   \n",
       "004       462.813980       462.813980   1613.322815   1613.322815   \n",
       "005       462.004196       462.004196   1568.070068   1568.070068   \n",
       "..               ...              ...           ...           ...   \n",
       "096       468.626633       467.077530   1980.721741   1980.721741   \n",
       "097       465.489029       465.489029   2349.541382   2349.541382   \n",
       "098       462.685913       467.453308   1914.418884   1914.418884   \n",
       "099       472.083267       466.991913   2254.812256   2254.812256   \n",
       "100       360.690430       360.690430   1791.389404   1791.389404   \n",
       "\n",
       "     T48PT_Pacific Blue-A_q12  T48PT_Pacific Blue-A_q34  ...  \\\n",
       "001                233.502617                233.502617  ...   \n",
       "002                186.389404                186.389404  ...   \n",
       "003                200.654549                200.654549  ...   \n",
       "004                212.288345                212.288345  ...   \n",
       "005                195.807983                195.807983  ...   \n",
       "..                        ...                       ...  ...   \n",
       "096                213.936401                213.936401  ...   \n",
       "097                203.944809                203.944809  ...   \n",
       "098                232.355072                232.355072  ...   \n",
       "099                203.208473                203.208473  ...   \n",
       "100                203.381996                203.381996  ...   \n",
       "\n",
       "     NKTP_PE-Cy7-A_q13  NKTP_PE-Cy7-A_q24  CD4_AmCyan-A_q12  CD4_AmCyan-A_q34  \\\n",
       "001        1073.911011        1078.053223        487.012085        498.808182   \n",
       "002       11729.672852        1558.832581        590.344879        594.977478   \n",
       "003        2032.824158         760.268158        587.791046        591.941376   \n",
       "004        2025.792664        1355.437561        596.772156        596.772156   \n",
       "005        2566.633301        1253.903687        402.203644        402.203644   \n",
       "..                 ...                ...               ...               ...   \n",
       "096        2906.513062        1249.622498        460.437057        457.869614   \n",
       "097        1305.977539        1294.955444        465.489029        465.489029   \n",
       "098        3014.272339        1272.894592        477.395081        477.395081   \n",
       "099        3049.025513        1068.188843        481.194122        484.080368   \n",
       "100        3031.520630         907.434998        464.351974        464.351974   \n",
       "\n",
       "     CD4_Pacific Blue-A_q13  CD4_Pacific Blue-A_q24  CD8_AmCyan-A_q12  \\\n",
       "001              211.248085              211.248085        504.310776   \n",
       "002              312.575684              312.575684        490.223831   \n",
       "003              303.369293              303.369293        471.380859   \n",
       "004              212.790939              212.790939        484.688721   \n",
       "005              206.456284              206.456284        482.189163   \n",
       "..                      ...                     ...               ...   \n",
       "096              200.150414              200.150414        427.757172   \n",
       "097              144.499069              144.499069        440.895248   \n",
       "098              155.943855              155.943855        371.646011   \n",
       "099              169.896675              169.896675        423.866104   \n",
       "100              167.162354              167.162354        296.643661   \n",
       "\n",
       "     CD8_AmCyan-A_q34  CD8_PerCP-A_q13  CD8_PerCP-A_q24  \n",
       "001        487.861938       990.332001      3825.442261  \n",
       "002        490.223831      1201.658386      6921.157471  \n",
       "003        479.999893      1184.574951      3082.143188  \n",
       "004        492.254013      1183.739258      2999.500122  \n",
       "005        487.244675      1170.567078      3372.401001  \n",
       "..                ...              ...              ...  \n",
       "096        432.122986      1040.649231      3051.995850  \n",
       "097        440.895248      1013.062683      2920.687866  \n",
       "098        371.646011       825.394165      2151.710449  \n",
       "099        423.866104       932.293732      3027.066162  \n",
       "100        296.643661       651.142914      2955.517822  \n",
       "\n",
       "[100 rows x 32 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gating_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bba79bcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "gating_df.to_pickle(pathlib.Path(r'data/processed_data/manual_gating_expert1.pkl'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}