Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1b267dab-64a3-485c-9c7b-19ec7f9cc46c",
   "metadata": {
    "tags": []
   },
   "source": [
    "# packages and imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "77526606-9bbf-495b-8ac7-e48be9cdc31e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-08-07 17:35:01.343135: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-08-07 17:35:01.501028: I tensorflow/core/util/util.cc:169] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-08-07 17:35:01.507459: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2023-08-07 17:35:01.507481: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "2023-08-07 17:35:01.538700: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2023-08-07 17:35:13.025540: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-08-07 17:35:13.025683: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-08-07 17:35:13.025698: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
      "2023-08-07 17:35:31.114481: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2023-08-07 17:35:31.114541: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2023-08-07 17:35:31.114580: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (sdx44): /proc/driver/nvidia/version does not exist\n"
     ]
    }
   ],
   "source": [
    "#imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from data_makers import *\n",
    "import utils\n",
    "from utils import *\n",
    "import mean_model\n",
    "from mean_model import meanModel\n",
    "import ML_models\n",
    "from ML_models import *\n",
    "\n",
    "import tensorflow as tf\n",
    "from alibi.explainers import IntegratedGradients\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from joblib import parallel_backend\n",
    "\n",
    "from scipy.stats.stats import pearsonr\n",
    "\n",
    "import shap\n",
    "\n",
    "from Nikhil_code import data_preprocessing as dp_nb\n",
    "from Nikhil_code import model_selection as ms_nb\n",
    "from Nikhil_code import testing as t_nb\n",
    "import Nikhil_code.Data_imports as di_nb\n",
    "import Nikhil_code.pairs_train_test_split as tts_nb\n",
    "import Nikhil_code.Learning_curve as lc_nb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a935de8c-6d7a-4c52-b03a-c41dd80efb90",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## imaging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3f66b3e-b26a-4774-9212-faaab11964df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# NN imaging imports\n",
    "\n",
    "from skimage import data, io   # Import skimage library (data - Test images and example data.\n",
    "#                          io - Reading, saving, and displaying images.)\n",
    "from skimage.color import rgb2gray\n",
    "\n",
    "import matplotlib.pyplot as plt                  # Import matplotlib.pyplot (Plotting framework in Python.)\n",
    "%matplotlib inline\n",
    "import os                                        # This module provides a portable way of using operating system dependent functionality.\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "import warnings\n",
    "# Suppress warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from IPython.display import display\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras import utils\n",
    "from tensorflow.keras.models import Sequential,load_model\n",
    "from tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import tensorflow as tf\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb98f7fb-4bcf-4073-9b72-a162a2e2f00f",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Feature selection (FS) and data creating for each drug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2864c18e-e967-48ca-97d7-1a1167f8cda7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Falling back to the 'python' engine because the 'c' engine does not support skipfooter; you can avoid this warning by specifying engine='python'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of missing prot values 0.386335609896865\n",
      "num non overlapping prot and target cls: 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Falling back to the 'python' engine because the 'c' engine does not support skipfooter; you can avoid this warning by specifying engine='python'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num non overlapping cls: 930\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((38, 8457), (38, 17417), (38, 22804), (38, 38), (345, 345))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#input data\n",
    "prot, rna, phospho_ls, one_hot_cls, one_hot_drugs, ic50_df1 = di_nb.read_input_data()\n",
    "_all_cls = prot.index\n",
    "_all_drugs = ic50_df1.columns\n",
    "assert prot.shape[0] == rna.shape[0] == phospho_ls.shape[0]\n",
    "assert phospho_ls.shape[0]  == one_hot_cls.shape[0]\n",
    "prot.shape, rna.shape, phospho_ls.shape, one_hot_cls.shape, one_hot_drugs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a1ae49c9-4f4f-4cb6-ab3c-4fc32dcb8724",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11583, 2056)\n",
      "(11583, 2401)\n"
     ]
    }
   ],
   "source": [
    "## PHOSPHO\n",
    "landmark_genes = pd.read_csv(\n",
    "    f'data/Landmark_genes_LINCS.txt',sep='\\t')\n",
    "landmark_genes.index = landmark_genes['Symbol']\n",
    "\n",
    "X_L1000 = landmark_X_maker(phospho_ls, landmark_genes['Symbol'])\n",
    "\n",
    "#create prot data for all drugs\n",
    "x_all_phos, x_drug, y_main_phos = dp_nb.create_all_drugs(\n",
    "    X_L1000, one_hot_drugs, ic50_df1, _all_cls)\n",
    "\n",
    "#fmt index to include drug cell line paris\n",
    "cls_drugs_index = x_all_phos.index + '::' + x_drug.index \n",
    "x_all_phos.index = cls_drugs_index\n",
    "y_main_phos.index = cls_drugs_index\n",
    "x_drug.index = cls_drugs_index\n",
    "\n",
    "x_all_phos = x_all_phos.astype(np.float32)\n",
    "print(x_all_phos.shape)\n",
    "X_main_phos = X_main_maker(x_all_phos, x_drug, short = False)\n",
    "print(X_main_phos.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0323541f-b188-456c-8e85-a06420bac4a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "##PROTEOMIC\n",
    "#use the same landmark genes for fs with prot data\n",
    "landmark_genes = pd.read_csv(\n",
    "    f'../data/Landmark_genes_LINCS.txt',sep='\\t')\n",
    "landmark_genes.index = landmark_genes['Symbol']\n",
    "\n",
    "#find overlapping landmark genes and prot features\n",
    "overlapping_landmarks, _ = dp_nb.keep_overlapping(\n",
    "    pd.DataFrame(landmark_genes['Symbol']), prot.T)\n",
    "\n",
    "overlapping_landmarks = overlapping_landmarks['Symbol'].values\n",
    "\n",
    "#create prot data for all drugs\n",
    "x_all_prot, x_drug, y_list = dp_nb.create_all_drugs(\n",
    "    prot[overlapping_landmarks], one_hot_drugs, ic50_df1, _all_cls)\n",
    "\n",
    "#fmt index to include drug cell line paris\n",
    "cls_drugs_index = x_all_prot.index + '::' + x_drug.index \n",
    "x_all_prot.index = cls_drugs_index\n",
    "y_list.index = cls_drugs_index\n",
    "x_drug.index = cls_drugs_index\n",
    "\n",
    "x_all_prot = x_all_prot.astype(np.float32)\n",
    "x_all_prot.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2f3769c8-5c2c-4cce-afe8-64cf95100861",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11583, 38)\n",
      "(11583, 383)\n"
     ]
    }
   ],
   "source": [
    "#one hot data creation for all drugs\n",
    "x_hot, x_drug_hot, y_main_onehot = dp_nb.create_all_drugs(\n",
    "    one_hot_cls, one_hot_drugs, ic50_df1, _all_cls)\n",
    "\n",
    "cls_drugs_index_hot = x_hot.index + '::' + x_drug_hot.index \n",
    "\n",
    "x_hot.index = cls_drugs_index_hot\n",
    "x_hot.columns = np.arange(len(x_drug.columns), len(x_drug.columns) + len(x_hot.columns))\n",
    "print(x_hot.shape)\n",
    "X_main_onehot = X_main_maker(x_hot, x_drug, short = False)\n",
    "print(X_main_onehot.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dde60b1e-ef70-43c6-b3bd-a6e52b663742",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Early stopping and parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "53338cbd-9ae8-4b45-bf86-f611424a92f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set early stopping and learning rate scheduler\n",
    "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20, min_delta=0, restore_best_weights=True)\n",
    "lr_scheduler = keras.callbacks.LearningRateScheduler(scheduler)\n",
    "callbacks_list =  [es,lr_scheduler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "810b7132-40e6-4d62-b989-7fac4ddd4621",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set parameters and train\n",
    "num_epochs = 200 # max epoch for training\n",
    "learning_rate = 1e-2 # initial learning rate, decays via learning rate scheduler\n",
    "momentum = 0.5\n",
    "initializer_seed = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc330970-e81a-48f7-8f49-255832af109f",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Random Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2e4a010-b448-4ec2-9fd9-589deff48206",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create dataframes that take a random set of features with the same length as landmark as a comparison\n",
    "random_X = X_phos.sample(n=2056,axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f706cf1-5867-4e26-b840-7162955b235d",
   "metadata": {},
   "outputs": [],
   "source": [
    "## PHOSPHO\n",
    "#create prot data for all drugs\n",
    "x_all_phos, x_drug, y_main_phos = dp_nb.create_all_drugs(\n",
    "    random_X, one_hot_drugs, ic50_df1, _all_cls)\n",
    "\n",
    "#fmt index to include drug cell line paris\n",
    "cls_drugs_index = x_all_phos.index + '::' + x_drug.index \n",
    "x_all_phos.index = cls_drugs_index\n",
    "y_main_phos.index = cls_drugs_index\n",
    "x_drug.index = cls_drugs_index\n",
    "\n",
    "x_all_phos = x_all_phos.astype(np.float32)\n",
    "print(x_all_phos.shape)\n",
    "X_main_phos = X_main_maker(x_all_phos, x_drug, short = False)\n",
    "print(X_main_phos.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2d2df83-cdf2-4b64-b66c-404303009337",
   "metadata": {},
   "outputs": [],
   "source": [
    "#make the model a classifier\n",
    "y_main_phos = classyFire(y_main_phos)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2beefc8-d363-41cd-b3ab-e8275382864b",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22cbf0aa-bdd4-4ca1-b864-de91eca82d1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#read in the feature_importances_ feature selected data\n",
    "feature_list = []\n",
    "with open(\"feat_select_files/phospho/rfr_X_main/rfr_feat_select.txt\", \"r\") as features:\n",
    "    lines = features.readlines()\n",
    "    for i in lines:\n",
    "        i.replace(\" \", \"\")\n",
    "        feature_list.append(i.split(\":\")[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53163c96-f9f1-442a-8c0c-548606bf6f8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a new X dataframe with the BEST selected features\n",
    "X_features = phospho_ls.reindex(feature_list[:2000],axis=\"columns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10c44e08-203a-4d8b-962b-6bcdd271c37f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a new X dataframe with the WORST selected features\n",
    "X_features = X_phos.reindex(feature_list[21785:22785],axis=\"columns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "beeec04e-12a1-4565-8948-1f0de4d5f15c",
   "metadata": {},
   "outputs": [],
   "source": [
    "## PHOSPHO\n",
    "#create prot data for all drugs\n",
    "x_all_phos, x_drug, y_main_phos = dp_nb.create_all_drugs(\n",
    "    X_features, one_hot_drugs, ic50_df1, _all_cls)\n",
    "\n",
    "#fmt index to include drug cell line paris\n",
    "cls_drugs_index = x_all_phos.index + '::' + x_drug.index \n",
    "x_all_phos.index = cls_drugs_index\n",
    "y_main_phos.index = cls_drugs_index\n",
    "x_drug.index = cls_drugs_index\n",
    "\n",
    "x_all_phos = x_all_phos.astype(np.float32)\n",
    "print(x_all_phos.shape)\n",
    "X_main_phos = X_main_maker(x_all_phos, x_drug, short = False)\n",
    "print(X_main_phos.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81be8c1c-caac-4e13-af44-4fb292a64a35",
   "metadata": {
    "tags": []
   },
   "source": [
    "# train-test-split"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5da3f45e-4f92-4ca6-8065-e9441bf94ba6",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Early integration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "94a673c0-8f40-4930-a224-d26048751adc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#regular split\n",
    "X_train, X_test, y_train, y_test = cell_line_split(X_main_phos, y_main_phos, test_size=0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5aa187b8-ba53-43e6-9dfc-61e625b5b511",
   "metadata": {},
   "outputs": [],
   "source": [
    "#short split\n",
    "X_train_short, X_test_short, y_train_short, y_test_short = cell_line_split(X_main_phos[:1000], y_main_phos[:1000], test_size=0.2, random_state = 8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "097c927a-29e7-4616-8c6d-cf38d9453b2a",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Late integration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b936cad8-b36d-4e60-9273-4115daddae23",
   "metadata": {},
   "outputs": [],
   "source": [
    "#phospho\n",
    "xo_train, xo_test = cell_line_split(x_all_phos, y=None, test_size=0.2, random_state = 0)\n",
    "xd_train, xd_test = cell_line_split(x_drug, y=None, test_size=0.2, random_state = 0)\n",
    "y_train, y_test = cell_line_split(y_main_phos, y=None, test_size=0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cda9a57-c6bd-4573-b922-a90abc84ab53",
   "metadata": {},
   "outputs": [],
   "source": [
    "#proteomic\n",
    "xo_train, xo_test = cell_line_split(x_all_prot, y=None, test_size=0.2, random_state = 0)\n",
    "xd_train, xd_test = cell_line_split(x_drug_prot, y=None, test_size=0.2, random_state = 0)\n",
    "y_train, y_test = cell_line_split(y_main_prot, y=None, test_size=0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c843d3f-a1e3-40f1-9cd3-6c9c2ba8aceb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#FOR TESTING NEW NETWORKS\n",
    "\n",
    "x_all_phos = x_all_phos[:100].T[:100]\n",
    "x_all_phos = x_all_phos.T\n",
    "\n",
    "x_drug_phos = x_drug_phos[:100]\n",
    "\n",
    "y_main_phos = y_main_phos[:100]\n",
    "\n",
    "xo_train, xo_test = cell_line_split(x_all_phos, y=None, test_size=0.2, random_state = 0)\n",
    "xd_train, xd_test = cell_line_split(x_drug_phos, y=None, test_size=0.2, random_state = 0)\n",
    "y_train, y_test = cell_line_split(y_main_phos, y=None, test_size=0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90f72e25-bc7e-4f49-b57f-953d2188909d",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Convolutional Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94d9e26c-6a00-4522-a90b-d76cb4b1e5ac",
   "metadata": {},
   "source": [
    "model building and training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6704b22d-15cf-4f7e-9c35-eaefceaee20c",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Test models here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29142bca-6e3a-40ae-8668-afb6d9c9f607",
   "metadata": {},
   "outputs": [],
   "source": [
    "## CNN using functional API\n",
    "def build_early_integration_DNN_short(X_train, learning_rate, momentum, seed, mtype='regression'):\n",
    "\n",
    "    # set layer weights initialiser\n",
    "    initializer = keras.initializers.GlorotUniform(seed=seed)\n",
    "    \n",
    "    # drug-cell line data input\n",
    "    x_input = layers.Input(shape=(X_train.shape[1]))\n",
    "\n",
    "    # 5 fully connected layers with batch norm and dropout\n",
    "    x = layers.Dense(16, kernel_initializer=initializer, activation='relu')(x_input)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "    x = layers.Dropout(0.1)(x)\n",
    "    x = layers.Dense(32, kernel_initializer=initializer, activation='relu')(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "    x = layers.Dropout(0.1)(x)\n",
    "    #x = layers.Dense(64, kernel_initializer=initializer, activation='relu')(x)\n",
    "    #x = layers.BatchNormalization()(x)\n",
    "    #x = layers.Dropout(0.1)(x)\n",
    "    x = layers.Dense(32, kernel_initializer=initializer, activation='relu')(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "    x = layers.Dropout(0.1)(x)\n",
    "    x = layers.Dense(16, kernel_initializer=initializer, activation='relu')(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "    x = layers.Dropout(0.1)(x)\n",
    "    \n",
    "    if mtype == 'classifier':\n",
    "        output = layers.Dense(3, activation = 'softmax',activity_regularizer=keras.regularizers.l2())(x) # actiation layer is 3 neuron for each class\n",
    "\n",
    "        model = keras.Model(x_input, output)\n",
    "\n",
    "        model.compile( loss='sparse_categorical_crossentropy', optimizer=keras.optimizers.Adam(), metrics=['accuracy'])\n",
    "    \n",
    "    if mtype == 'regression':\n",
    "        output = layers.Dense(1, kernel_initializer=initializer)(x) # actiation layer is 1 neuron for single regression prediction value\n",
    "        \n",
    "        model = keras.Model(x_input, output)\n",
    "    \n",
    "        model.compile(optimizer=keras.optimizers.RMSprop(learning_rate=learning_rate, \n",
    "                                                        momentum=momentum), \n",
    "                                                        loss='mse', metrics=['mae'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e01c6135-4433-4723-9ab8-882a7a0ba5e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = build_early_integration_DNN_short(X_train, learning_rate, momentum, initializer_seed, mtype='regression')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30a3f131-de56-46f7-8a3f-30071ee74855",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9bb5b97-8fe0-45da-a760-80ab2d43b45e",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Early integration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "706ba73d-c278-45d5-a1eb-fc68c4b7f96f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'X_train shape: {X_train.shape}\\ny_train shape: {y_train.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea5ddb4e-62c6-49e1-b94f-2cb045e17f86",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "## Code for model building and training \n",
    " \n",
    "model = build_early_integration_DNN(X_train, learning_rate, momentum, initializer_seed, mtype='regression')\n",
    "history = model.fit(X_train, y_train,\n",
    "                        validation_data=(X_test, y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "y_pred = model.predict(X_test)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d588fcd-52f2-4f20-b2eb-8cbc4787895f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'length of y_pred: {len(y_pred)}\\nlength of y_test:{len(y_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06d9fe94-d5ba-4c0a-bb6d-3853b63136d0",
   "metadata": {},
   "source": [
    "regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e315a03f-65b6-42a1-84fb-9364bffaf5ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "#regression metrics\n",
    "\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "MSE = mean_squared_error(y_test, y_pred)\n",
    "pearson = pearsonr(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4e7a7bc-a0bc-4297-8242-1692fd12d3eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'r-squared: {r2}\\nMSE: {MSE}\\npearson: {pearson}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eaf1e017-c9e0-4570-9cb4-5a73dd459d13",
   "metadata": {},
   "source": [
    "classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3cb96fd-ef7d-47b5-bd54-4bdb68644052",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_dict = {0:'High responsiveness', 1:'Intermediate responsiveness', 2:'Low responsiveness'}\n",
    "class_list = [0, 1, 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a1b8f25-b9ab-405f-841e-ce88bf39f774",
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 3026\n",
    "print(f'For the screening of {y_test.index[idx]}\\nThe prediction is:- high: {y_pred[idx][0]}, intermediate:- {y_pred[idx][1]}, low:- {y_pred[idx][2]}\\nThe true value is: {y_test[idx]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0231024-269a-40fb-b874-e220e32a8a82",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = tf.keras.metrics.SparseCategoricalAccuracy()\n",
    "m.update_state(y_test, y_pred)\n",
    "m.result().numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "243bbbc0-bc40-4fd2-8700-0b1018846d59",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Late integration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cd76fd6-2b18-4ded-aade-a7b7921b9ac0",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'xo_train shape: {xo_train.shape}\\nxd_train shape: {xd_train.shape}\\ny_train shape: {y_train.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4dfbe30f-6c69-4511-8c82-14567ae6b84a",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "## Code for model building and training \n",
    " \n",
    "model = build_Deep_NN_tester(xo_train, xd_train, learning_rate, momentum, initializer_seed, complexity = 5)\n",
    "history = model.fit([xo_train, xd_train], y_train,\n",
    "                        validation_data=([xo_test, xd_test], y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "y_pred = model.predict([xo_test, xd_test])  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66c84d46-f1d9-430b-8a3f-95e73fef1b74",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'length of y_pred: {len(y_pred)}\\nlength of y_test:{len(y_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dde86b58-15af-42c6-b1d3-d2c436c7bea4",
   "metadata": {},
   "source": [
    "regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfeca954-8b18-4eb8-98f5-329aae1fd074",
   "metadata": {},
   "outputs": [],
   "source": [
    "r2 = r2_score(y_test, y_pred)\n",
    "MSE = mean_squared_error(y_test, y_pred)\n",
    "pearson = pearsonr(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "674936e3-fbcc-4909-beae-2582424fe7ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'r-squared: {r2}\\nMSE: {MSE}\\npearson: {pearson}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6eb07663-a802-41ab-a08b-16d24a0c66f7",
   "metadata": {},
   "source": [
    "classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "276eb83c-8d1c-4289-aab8-f67127172300",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_dict = {0:'High responsiveness', 1:'Intermediate responsiveness', 2:'Low responsiveness'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adf4ede9-eba8-4d01-a60a-bf615ea7f9e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'For the screening of {y_test.index[0]}\\nThe prediction is:- high: {y_pred[0][0]}, intermediate:- {y_pred[0][1]}, low:- {y_pred[0][2]}\\nThe true value is: {y_test[0]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f9abb1f-5be6-4968-b299-c360a6fa8bcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = tf.keras.metrics.SparseCategoricalAccuracy()\n",
    "m.update_state(y_test, y_pred)\n",
    "m.result().numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9b70559-18c7-4168-9b38-b2110ea50a05",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Visualise networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b948d59a-4982-4a38-b7d4-01395c825477",
   "metadata": {},
   "outputs": [],
   "source": [
    "#LATE INTEGRATION\n",
    "\n",
    "learning_rate = 1e-2 # initial learning rate, decays via learning rate scheduler\n",
    "momentum = 0.5\n",
    "initializer_seed = 0\n",
    " \n",
    "late_model = build_CNN(xo_train, xd_train, learning_rate, momentum, initializer_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d0902ba-4cf6-4eb4-a92f-8b9aea9aef32",
   "metadata": {},
   "outputs": [],
   "source": [
    "#EARLY INTEGRATION\n",
    "\n",
    "learning_rate = 1e-2 # initial learning rate, decays via learning rate scheduler\n",
    "momentum = 0.5\n",
    "initializer_seed = 42\n",
    " \n",
    "early_model = build_early_integration_CNN(X_train, learning_rate, momentum, initializer_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18e1746b-7479-458f-a34b-16a8d75d5850",
   "metadata": {},
   "outputs": [],
   "source": [
    "late_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "065864c9-6559-4330-a493-1498b8f01c2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "early_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5290a9c5-6c6f-46b6-b239-bcfc88e45378",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ann_visualizer.visualize import ann_viz\n",
    "\n",
    "ann_viz(early_model, view=True, filename='construct_model', title='CNN — Model 1 — Simple Architecture')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b967d6f-b279-4822-8cf9-9edbe4c55b8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import visualkeras\n",
    "from PIL import ImageFont\n",
    "\n",
    "visualkeras.graph_view(early_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a396d39d-43f0-49f8-8a87-f1ddb2707c40",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext tensorboard\n",
    "from datetime import datetime\n",
    "from tensorflow import keras\n",
    "\n",
    "# Define the Keras TensorBoard callback.\n",
    "logdir=\"logs/fit/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcfe6cf7-0f01-45b2-876c-d3ce8901b49e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set early stopping and learning rate scheduler\n",
    "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20, min_delta=0, restore_best_weights=True)\n",
    "lr_scheduler = tf.keras.callbacks.LearningRateScheduler(scheduler)\n",
    "\n",
    "history = early_model.fit(X_train_short, y_train_short,\n",
    "                        validation_data=(X_test_short, y_test_short),\n",
    "                        epochs=200, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=[tensorboard_callback, es, lr_scheduler])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfa8be32-02d7-419c-b467-2fd7ca8a558b",
   "metadata": {},
   "outputs": [],
   "source": [
    "logdir"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4619359-e738-4462-858a-0ed3a3e0e3f8",
   "metadata": {
    "tags": []
   },
   "source": [
    "## phospho"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b29e9011-b73b-41e6-ab3e-81c55c201c4e",
   "metadata": {},
   "source": [
    "Late integration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62588747-cdf6-4e73-bcde-dd2f56c45d35",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'X_train shape: {xo_train.shape}\\ny_train shape: {y_train.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8125661b-7fd6-493c-8362-c3168c39ae5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_all_phos[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "096a94a2-c01c-4d83-89e7-5ae4e54d54d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_main_phos[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4f4e546-28d8-465e-b720-61147e0aeb33",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "rs = [0, 8, 23, 42, 69, 88]\n",
    "mm_r2 = []\n",
    "mixed_set_r2 = []\n",
    "model_r2 = []\n",
    "model_MSE = []\n",
    "model_pearson = []\n",
    "\n",
    "for seed in rs:\n",
    "    \n",
    "    print(f'Seed: {seed}')\n",
    "    \n",
    "    print(f'Mean Model')\n",
    "    #MEAN MODEL\n",
    "    #tts\n",
    "    X_train, X_test, y_train, y_test = cell_line_split(X_main_phos, y=y_main_phos, test_size=0.2, random_state = seed)\n",
    "    #run mean model\n",
    "    dl = dlMaker(y_train, noRepeats=True)\n",
    "    mm = meanModel(y_train, dl)\n",
    "    prediction = mm.predict(y_test.index)\n",
    "    mean_r2 = r2_score(y_test, list(prediction.values()))\n",
    "    mm_r2.append(mean_r2)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    print(f'Mixed-Set Model')\n",
    "    #MIXED SET\n",
    "    # one hot the dataframes\n",
    "    onehotX = one_hot_maker(X_L1000, X_df=True)\n",
    "    \n",
    "    #produce X-main and y_main\n",
    "    cl = clMaker(onehotX, y_phos)\n",
    "    x_all_oneHot, x_drug_oneHot, y_main_oneHot = create_all_drugs(x=onehotX, xd=hotdrugsDF_phos, y=y_phos, cells=cl)\n",
    "    \n",
    "    #phospho\n",
    "    xo_train, xo_test = cell_line_split(x_all_oneHot, y=None, test_size=0.2, random_state = seed)\n",
    "    xd_train, xd_test = cell_line_split(x_drug_oneHot, y=None, test_size=0.2, random_state = seed)\n",
    "    y_train, y_test = cell_line_split(y_main_oneHot, y=None, test_size=0.2, random_state = seed)\n",
    "    \n",
    "    ## Code for model building and training\n",
    "\n",
    "    model = build_Deep_NN(xo_train, xd_train, learning_rate, momentum, initializer_seed, complexity = 1)\n",
    "    history = model.fit([xo_train, xd_train], y_train,\n",
    "                        validation_data=([xo_test, xd_test], y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "    y_pred = model.predict([xo_test, xd_test])  \n",
    "    r2 = r2_score(y_test, y_pred)\n",
    "        \n",
    "    mixed_set_r2.append(r2)\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    print(f'Model: CNN')\n",
    "    \n",
    "    #phospho\n",
    "    xo_train, xo_test = cell_line_split(x_all_phos, y=None, test_size=0.2, random_state = seed)\n",
    "    xd_train, xd_test = cell_line_split(x_drug_phos, y=None, test_size=0.2, random_state = seed)\n",
    "    y_train, y_test = cell_line_split(y_main_phos, y=None, test_size=0.2, random_state = seed)\n",
    "    \n",
    "\n",
    "    ## Code for model building and training\n",
    "\n",
    "    model = build_Deep_NN(xo_train, xd_train, learning_rate, momentum, initializer_seed, complexity = 1)\n",
    "    history = model.fit([xo_train, xd_train], y_train,\n",
    "                        validation_data=([xo_test, xd_test], y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "    y_pred = model.predict([xo_test, xd_test])  \n",
    "    r2 = r2_score(y_test, y_pred)\n",
    "    MSE = mean_squared_error(y_test, y_pred)\n",
    "    pearson = pearsonr(y_test, y_pred)\n",
    "    model_r2.append(r2)\n",
    "    model_MSE.append(MSE)\n",
    "    model_pearson.append(pearson)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e90d5d37-a6ea-45ea-aa84-90d1b7382ab7",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_make('CNN_metrics',\n",
    "           [rs, mm_r2, mixed_set_r2, model_r2, model_MSE, model_pearson],\n",
    "           ['rs', 'mm_r2', 'mixed_set_r2', 'model_r2', 'model_MSE', 'model_pearson'], \n",
    "           file = 'plots/phospho/NormalisedX_LateDNN(1)_metrics_L1000.csv')\n",
    "           "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08d38aa3-7b5f-434e-9539-ac3c3d8828c2",
   "metadata": {},
   "source": [
    "Early integration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "accff2e0-bfce-4810-b1c7-f13f7ce052eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'X_train shape: {X_train.shape}\\ny_train shape: {y_train.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3a49430-0595-417d-b8c0-b1f7d046a259",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "rs = [0, 8, 23, 42, 69, 88]\n",
    "mm_r2 = []\n",
    "mixed_set_r2 = []\n",
    "model_r2 = []\n",
    "model_MSE = []\n",
    "model_pearson = []\n",
    "\n",
    "for seed in rs:\n",
    "    \n",
    "    print(f'Seed: {seed}')\n",
    "    \n",
    "    print(f'Mean Model')\n",
    "    #MEAN MODEL\n",
    "    #tts\n",
    "    X_train, X_test, y_train, y_test = cell_line_split(X_main_phos, y=y_main_phos, test_size=0.2, random_state = seed)\n",
    "    #run mean model\n",
    "    dl = dlMaker(y_train, noRepeats=True)\n",
    "    mm = meanModel(y_train, dl)\n",
    "    prediction = mm.predict(y_test.index)\n",
    "    mean_r2 = r2_score(y_test, list(prediction.values()))\n",
    "    mm_r2.append(mean_r2)\n",
    "    \n",
    "    print(f'Mixed-Set Model')\n",
    "    #MIXED SET\n",
    "    # one hot the dataframes\n",
    "    onehotX = one_hot_maker(X_L1000, X_df=True)\n",
    "    \n",
    "    #produce X-main and y_main\n",
    "    cl = clMaker(onehotX, y_phos)\n",
    "    x_all_oneHot, x_drug_oneHot, y_main_oneHot = create_all_drugs(x=onehotX, xd=hotdrugsDF_phos, y=y_phos, cells=cl)\n",
    "    X_main_oneHot = X_main_maker(x_all_oneHot, x_drug_oneHot, short = False)\n",
    "    \n",
    "    #regular split\n",
    "    X_train, X_test, y_train, y_test = cell_line_split(X_main_oneHot, y_main_oneHot, test_size=0.2, random_state = 0)\n",
    "\n",
    "    ## Code for model building and training\n",
    "    \n",
    "    model = DeepIC50(X_train, learning_rate, momentum, initializer_seed, mtype='regression')\n",
    "    history = model.fit(X_train, y_train,\n",
    "                        validation_data=(X_test, y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "    y_pred = model.predict(X_test)  \n",
    "    \n",
    "    r2 = r2_score(y_test, y_pred)\n",
    "    mixed_set_r2.append(r2)\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    print(f'Model: CNN')\n",
    "    \n",
    "    #regular split\n",
    "    X_train, X_test, y_train, y_test = cell_line_split(X_main_phos, y_main_phos, test_size=0.2, random_state = 0)\n",
    "\n",
    "    ## Code for model building and training\n",
    "\n",
    "    model = DeepIC50(X_train, learning_rate, momentum, initializer_seed, mtype='regression')\n",
    "    history = model.fit(X_train, y_train,\n",
    "                        validation_data=(X_test, y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "    y_pred = model.predict(X_test)  \n",
    "    r2 = r2_score(y_test, y_pred)\n",
    "    MSE = mean_squared_error(y_test, y_pred)\n",
    "    pearson = pearsonr(y_test, y_pred)\n",
    "    model_r2.append(r2)\n",
    "    model_MSE.append(MSE)\n",
    "    model_pearson.append(pearson)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15ceaa26-c7b0-4b06-8602-dce0ab8ae87f",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_make('CNN_metrics',\n",
    "           [rs, mm_r2, mixed_set_r2, model_r2, model_MSE, model_pearson],\n",
    "           ['rs', 'mm_r2', 'mixed_set_r2', 'model_r2', 'model_MSE', 'model_pearson'], \n",
    "           file = 'plots/phospho/NormalisedX_DeepIC50_metrics_L1000.csv')\n",
    "           "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db1dc290-d1ec-442d-84db-9b0862934758",
   "metadata": {
    "tags": []
   },
   "source": [
    "## proteomic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "746c16dd-d23b-40bd-9cc6-b590193d31a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "rs = [0, 8, 23, 42, 69, 88]\n",
    "mm_r2 = []\n",
    "mixed_set_r2 = []\n",
    "model_r2 = []\n",
    "model_MSE = []\n",
    "model_pearson = []\n",
    "\n",
    "for seed in rs:\n",
    "    \n",
    "    print(f'Seed: {seed}')\n",
    "    \n",
    "    print(f'Mean Model')\n",
    "    #MEAN MODEL\n",
    "    #tts\n",
    "    X_train, X_test, y_train, y_test = cell_line_split(X_main_prot, y=y_main_prot, test_size=0.2, random_state = seed)\n",
    "    #run mean model\n",
    "    dl = dlMaker(y_train, noRepeats=True)\n",
    "    mm = meanModel(y_train, dl)\n",
    "    prediction = mm.predict(y_test.index)\n",
    "    mean_r2 = r2_score(y_test, list(prediction.values()))\n",
    "    mm_r2.append(mean_r2)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    print(f'Mixed-Set Model')\n",
    "    #MIXED SET\n",
    "    # one hot the dataframes\n",
    "    onehotX = one_hot_maker(X_prot, X_df=True)\n",
    "    \n",
    "    #produce X-main and y_main\n",
    "    cl = clMaker(onehotX, y_prot)\n",
    "    x_all_oneHot, x_drug_oneHot, y_main_oneHot = create_all_drugs(x=onehotX, xd=hotdrugsDF_prot, y=y_prot, cells=cl)\n",
    "    \n",
    "    #phospho\n",
    "    xo_train, xo_test = cell_line_split(x_all_oneHot, y=None, test_size=0.2, random_state = seed)\n",
    "    xd_train, xd_test = cell_line_split(x_drug_oneHot, y=None, test_size=0.2, random_state = seed)\n",
    "    y_train, y_test = cell_line_split(y_main_oneHot, y=None, test_size=0.2, random_state = seed)\n",
    "\n",
    "    ## Code for model building and training\n",
    "\n",
    "    # set early stopping and learning rate scheduler\n",
    "    es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20, min_delta=0, restore_best_weights=True)\n",
    "    lr_scheduler = tf.keras.callbacks.LearningRateScheduler(scheduler)\n",
    "    callbacks_list =  [es,lr_scheduler]\n",
    "\n",
    "    # set parameters and train\n",
    "    num_epochs = 200 # max epoch for training\n",
    "    learning_rate = 1e-2 # initial learning rate, decays via learning rate scheduler\n",
    "    momentum = 0.5\n",
    "    initializer_seed = 42\n",
    "\n",
    "    model = build_Deep_NN(xo_train, xd_train, learning_rate, momentum, initializer_seed)\n",
    "    history = model.fit([xo_train, xd_train], y_train,\n",
    "                        validation_data=([xo_test, xd_test], y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "    y_pred = model.predict([xo_test, xd_test])  \n",
    "    r2 = r2_score(y_test, y_pred)\n",
    "        \n",
    "    mixed_set_r2.append(r2)\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    print(f'Model: CNN')\n",
    "    \n",
    "    #phospho\n",
    "    xo_train, xo_test = cell_line_split(x_all_prot, y=None, test_size=0.2, random_state = seed)\n",
    "    xd_train, xd_test = cell_line_split(x_drug_prot, y=None, test_size=0.2, random_state = seed)\n",
    "    y_train, y_test = cell_line_split(y_main_prot, y=None, test_size=0.2, random_state = seed)\n",
    "    \n",
    "\n",
    "    ## Code for model building and training\n",
    "\n",
    "    # set early stopping and learning rate scheduler\n",
    "    es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20, min_delta=0, restore_best_weights=True)\n",
    "    lr_scheduler = tf.keras.callbacks.LearningRateScheduler(scheduler)\n",
    "    callbacks_list =  [es,lr_scheduler]\n",
    "\n",
    "    # set parameters and train\n",
    "    num_epochs = 200 # max epoch for training\n",
    "    learning_rate = 1e-2 # initial learning rate, decays via learning rate scheduler\n",
    "    momentum = 0.5\n",
    "    initializer_seed = 42\n",
    "\n",
    "    model = build_Deep_NN(xo_train, xd_train, learning_rate, momentum, initializer_seed)\n",
    "    history = model.fit([xo_train, xd_train], y_train,\n",
    "                        validation_data=([xo_test, xd_test], y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "    y_pred = model.predict([xo_test, xd_test])  \n",
    "    r2 = r2_score(y_test, y_pred)\n",
    "    MSE = mean_squared_error(y_test, y_pred)\n",
    "    pearson = pearsonr(y_test, y_pred)\n",
    "    model_r2.append(r2)\n",
    "    model_MSE.append(MSE)\n",
    "    model_pearson.append(pearson)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c848717-4dae-4ff4-9f18-15eb4b11131e",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_make('Deep_NN_metrics_prot',\n",
    "           [rs, mm_r2, mixed_set_r2, model_r2, model_MSE, model_pearson],\n",
    "           ['rs', 'mm_r2', 'mixed_set_r2', 'model_r2', 'model_MSE', 'model_pearson'], \n",
    "           file = 'plots/proteomic/Deep_NN_metrics.csv')\n",
    "           "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e32a9d86-376e-46b8-8a0a-d80894f356b7",
   "metadata": {
    "tags": []
   },
   "source": [
    "## mixed set model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e9a93bb-9d73-48cb-8afe-7caeeb60e412",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Code for model building and training\n",
    "\n",
    "# set early stopping and learning rate scheduler\n",
    "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20, min_delta=0, restore_best_weights=True)\n",
    "lr_scheduler = tf.keras.callbacks.LearningRateScheduler(scheduler)\n",
    "callbacks_list =  [es,lr_scheduler]\n",
    "\n",
    "# set parameters and train\n",
    "num_epochs = 200 # max epoch for training\n",
    "learning_rate = 1e-2 # initial learning rate, decays via learning rate scheduler\n",
    "momentum = 0.5\n",
    "initializer_seed = 42\n",
    "\n",
    "model = build_Deep_NN(xo_train, xd_train, learning_rate, momentum, initializer_seed)\n",
    "history = model.fit([xo_train, xd_train], y_train,\n",
    "                    validation_data=([xo_test, xd_test], y_test),\n",
    "                    epochs=num_epochs, \n",
    "                    batch_size=None, \n",
    "                    verbose=0, \n",
    "                    callbacks=callbacks_list)\n",
    "\n",
    "y_pred = model.predict([xo_test, xd_test])  \n",
    "r2 = r2_score(y_test, y_pred)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9668cdd-4fe2-43a1-b2eb-1ce6506de2d3",
   "metadata": {
    "tags": []
   },
   "source": [
    "# SHAP for NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a06183b9-a3e8-4c4d-a36c-e02e495cb60e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (9009, 2056)\n",
      "X_train shape: (9009, 345)\n",
      "y_train shape: (9009,)\n"
     ]
    }
   ],
   "source": [
    "print(f'X_train shape: {xo_train.shape}')\n",
    "print(f'X_train shape: {xd_train.shape}')\n",
    "print(f'y_train shape: {y_train.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "549147b9-ffee-4d2b-8a65-6ec970bc6299",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-08-07 17:38:13.754196: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restoring model weights from the end of the best epoch: 28.\n",
      "Epoch 48: early stopping\n",
      "CPU times: user 12min 35s, sys: 26.1 s, total: 13min 1s\n",
      "Wall time: 3min 30s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## fit the model\n",
    "\n",
    "model5 = build_CNN(xo_train, xd_train, learning_rate, momentum, initializer_seed, mtype = 'regression')\n",
    "history = model5.fit([xo_train, xd_train], y_train,\n",
    "                    validation_data=([xo_test, xd_test], y_test),\n",
    "                    epochs=num_epochs, \n",
    "                    batch_size=None, \n",
    "                    verbose=0, \n",
    "                    callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "508356fb-b988-4675-b8ce-7c1b3ac7c518",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# DeepExplainer to explain predictions of the model\n",
    "explainer = shap.GradientExplainer(model5, [xo_train, xd_train])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "321f3118-ad0e-4996-bb2c-70844583c1ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cell line: KYSE-450\n",
      "drug: (5Z)-7-Oxozeaenol\n",
      "IC50: 1.708192\n"
     ]
    }
   ],
   "source": [
    "#specify the cell line and drugs that are tested\n",
    "idx = 0\n",
    "\n",
    "cell_line, drug = list(xo_test.index)[idx].split('::')\n",
    "\n",
    "print(f'cell line: {cell_line}\\ndrug: {drug}\\nIC50: {y_test[f\"{cell_line}::{drug}\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e53d002f-a011-42c9-ab7b-85f04a6879d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# make a shap value array for the prediction\n",
    "shaps = explainer.shap_values([np.array([xo_test.iloc[0]]), np.array([xd_test.iloc[0]])])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed92ebf1-dac1-42f5-986c-335a9270d1a5",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Permutation testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc92353a-9ebb-4316-a7ad-588a97a1d6b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from alibi.explainers import PermutationImportance, plot_permutation_importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3039910e-2ed6-4027-8816-391dae6f84d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'X_train shape: {X_train.shape}\\ny_train shape: {y_train.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9aa3915d-f78a-451b-9033-4ba8d6e59131",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "## fit the model\n",
    "\n",
    "#build model\n",
    "model6 = build_early_integration_DNN_short(X_train, learning_rate, momentum, initializer_seed, mtype='regression')\n",
    "history = model6.fit(X_train, y_train,\n",
    "                        validation_data=(X_test, y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "345e07fc-fb50-4246-afb0-2fd09d9eb61b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_fn(X_train):\n",
    "    return model6.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c29701a9-5ed4-4184-9557-091da0ea1702",
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = PermutationImportance(predictor=predict_fn,\n",
    "                                  score_fns=['r2'],\n",
    "                                  feature_names=list(X_train.columns),\n",
    "                                  verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11b193f3-6e5e-4772-83aa-0e7a34e25533",
   "metadata": {},
   "outputs": [],
   "source": [
    "#specify the cell line and drugs that are tested\n",
    "idx = 0\n",
    "\n",
    "cell_line, drug = list(xo_test.index)[idx].split('::')\n",
    "\n",
    "print(f'cell line: {cell_line}\\ndrug: {drug}\\nIC50: {y_test[f\"{cell_line}::{drug}\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51dd34c4-32e2-48d8-8cf2-acdff3ea49ff",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "exp = explainer.explain(X=X_test.values, y=y_test.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "076934f5-18ec-48c3-a522-af62fac575f8",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "feat_importances = []\n",
    "for i, feat in enumerate(exp.feature_importance[0][:100]):\n",
    "    feat_importances.append(f'{exp.feature_names[:100][i]}: {list(feat.values())[0]}')\n",
    "feat_importances"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4de0baf2-4d2c-4dd9-923c-b0c7f0c57d9c",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Integrated Gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "a49bc43c-27f5-4e38-8fc0-72d790d87d28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (9009, 2401)\n",
      "y_train shape: (9009,)\n"
     ]
    }
   ],
   "source": [
    "print(f'X_train shape: {X_train.shape}\\ny_train shape: {y_train.shape}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa6482df-5eb5-4694-babf-42f69696e171",
   "metadata": {},
   "source": [
    "Early integration DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "abb7e40e-06f9-47a1-be1f-644fb4c86e23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restoring model weights from the end of the best epoch: 25.\n",
      "Epoch 45: early stopping\n",
      "CPU times: user 12min 12s, sys: 15.7 s, total: 12min 28s\n",
      "Wall time: 3min 15s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "#build model\n",
    "model1 = build_early_integration_DNN(X_train, learning_rate, momentum, initializer_seed, mtype='regression')\n",
    "\n",
    "#fit model\n",
    "history = model1.fit(X_train, y_train,\n",
    "                        validation_data=(X_test, y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "#initialise IG explainer\n",
    "ig1  = IntegratedGradients(model1,\n",
    "                          layer=None,\n",
    "                          target_fn=None,\n",
    "                          method=\"gausslegendre\",\n",
    "                          n_steps=50,\n",
    "                          internal_batch_size=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32319bf7-2f3f-4fa2-9723-5f76fb5dbf12",
   "metadata": {},
   "source": [
    "Early integration DeepIC50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2b3c0b74-b505-4448-88cc-b21657b2cc86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restoring model weights from the end of the best epoch: 57.\n",
      "Epoch 77: early stopping\n",
      "CPU times: user 1h 20min 48s, sys: 1min 39s, total: 1h 22min 28s\n",
      "Wall time: 20min 31s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "#build model\n",
    "model2 = DeepIC50(X_train, learning_rate, momentum, initializer_seed, mtype='regression')\n",
    "\n",
    "#fit model\n",
    "history = model2.fit(X_train, y_train,\n",
    "                        validation_data=(X_test, y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "#Initialise IG explainer\n",
    "ig2  = IntegratedGradients(model2,\n",
    "                          layer=None,\n",
    "                          target_fn=None,\n",
    "                          method=\"gausslegendre\",\n",
    "                          n_steps=50,\n",
    "                          internal_batch_size=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b034cb30-4825-4c1b-be2a-20c511c81bec",
   "metadata": {},
   "source": [
    "Late integration DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "8ba5c27e-08b3-4490-a498-0cf9c17bdab3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restoring model weights from the end of the best epoch: 15.\n",
      "Epoch 35: early stopping\n",
      "CPU times: user 1min 27s, sys: 6.03 s, total: 1min 33s\n",
      "Wall time: 40 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    " \n",
    "#build model\n",
    "model3 = build_Deep_NN(xo_train, xd_train, learning_rate, momentum, initializer_seed, complexity = 4)\n",
    "\n",
    "#fit model\n",
    "history = model3.fit([xo_train, xd_train], y_train,\n",
    "                        validation_data=([xo_test, xd_test], y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "#Initialise IG explainer\n",
    "ig3  = IntegratedGradients(model3,\n",
    "                          layer=None,\n",
    "                          target_fn=None,\n",
    "                          method=\"gausslegendre\",\n",
    "                          n_steps=50,\n",
    "                          internal_batch_size=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ee16879-64eb-438f-8378-d561e625f49b",
   "metadata": {},
   "source": [
    "Late integration CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "f81058b8-e6bb-4833-a249-9541b3f056dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restoring model weights from the end of the best epoch: 24.\n",
      "Epoch 44: early stopping\n",
      "CPU times: user 11min 6s, sys: 21.8 s, total: 11min 28s\n",
      "Wall time: 3min 4s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#build model\n",
    "model4 = build_CNN(xo_train, xd_train, learning_rate, momentum, initializer_seed, mtype = 'regression')\n",
    "\n",
    "#fit model\n",
    "history = model4.fit([xo_train, xd_train], y_train,\n",
    "                        validation_data=([xo_test, xd_test], y_test),\n",
    "                        epochs=num_epochs, \n",
    "                        batch_size=None, \n",
    "                        verbose=0, \n",
    "                        callbacks=callbacks_list)\n",
    "\n",
    "#Initialise IG explainer\n",
    "ig4  = IntegratedGradients(model4,\n",
    "                          layer=None,\n",
    "                          target_fn=None,\n",
    "                          method=\"gausslegendre\",\n",
    "                          n_steps=50,\n",
    "                          internal_batch_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "b53455d2-9bc3-412b-83f3-dd1ff458885f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cell line: ML-2\n",
      "drug: (5Z)-7-Oxozeaenol\n"
     ]
    }
   ],
   "source": [
    "idx = 2\n",
    "num_feats = 100\n",
    "\n",
    "#specify the cell line and drugs that are tested\n",
    "\n",
    "cell_line, drug = list(xo_test.index)[idx].split('::')\n",
    "\n",
    "print(f'cell line: {cell_line}\\ndrug: {drug}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d593c2db-1138-4383-88b5-dc52ee33adaf",
   "metadata": {},
   "source": [
    "Early integration DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0647073e-425f-47cf-8e24-ed425efcb081",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 157ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "It looks like you are passing a model with a scalar output and target is set to `None`.If your model is a regression model this will produce correct attributions. If your model is a classification model, targets for each datapoint must be defined. Not defining the target may lead to incorrect values for the attributions.Targets can be either the true classes or the classes predicted by the model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------\n",
      "True IC50 value: 0.948509\n",
      "Predicted IC50 value: 1.2860968112945557\n",
      "------------------------------------------------\n",
      "CDK1(S233);:0.01420361208843644\n",
      "MAP7(S365);:0.007983334899808334\n",
      "SPP1(S215);:0.00792561300044159\n",
      "MSH6(S43);:0.006931786660178099\n",
      "NOSIP(S107);:0.006218817333435036\n",
      "RAP1GAP(T486);:0.005021102113867307\n",
      "RAP1GAP(S484);:0.004924069337543032\n",
      "IGF2R(S2400);IGF2R(S2401);:0.004649320641185236\n",
      "RRP8(S51);RRP8(S53);:0.0037547182103572527\n",
      "KDM3A(S463);:0.0031338973673540236\n",
      "------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#IC50 prediction\n",
    "y_pred = model1.predict(X_test[idx:(idx+1)])  \n",
    "print('------------------------------------------------')\n",
    "print(f'True IC50 value: {y_train[idx:(idx+1)].values[0]}\\nPredicted IC50 value: {y_pred[0][0]}')\n",
    "\n",
    "#explainer instance\n",
    "explanation = ig1.explain(X_train[idx:(idx+1)].values,\n",
    "                         baselines=None,\n",
    "                         target=None)\n",
    "\n",
    "#print the top features for this explanation \n",
    "attributions = explanation.attributions\n",
    "#this function outputs the top x number of features and their scores for a model\n",
    "print('------------------------------------------------')\n",
    "NN_final_names, NN_final_scores = rfrFeatures(attributions[0][0], X_main = X_train, topX = num_feats, N = num_feats)\n",
    "for i in range(10):\n",
    "    print(f'{NN_final_names[i]}:{NN_final_scores[i]}')\n",
    "print('------------------------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02db828f-24fd-4273-977d-f3209ebbe8c2",
   "metadata": {},
   "source": [
    "Early integration DeepIC50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a6b99efe-a078-42a7-8acb-d40dd5bbab4a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 281ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "It looks like you are passing a model with a scalar output and target is set to `None`.If your model is a regression model this will produce correct attributions. If your model is a classification model, targets for each datapoint must be defined. Not defining the target may lead to incorrect values for the attributions.Targets can be either the true classes or the classes predicted by the model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------\n",
      "True IC50 value: 0.948509\n",
      "Predicted IC50 value: 2.1359477043151855\n",
      "------------------------------------------------\n",
      "CCDC86(S37);:50.08771652675226\n",
      "IGF2R(S2400);IGF2R(S2401);:47.53724513202866\n",
      "IGF2R(T2415);:44.248619555893484\n",
      "DDX42(S751);DDX42(S754);:27.72710106155702\n",
      "GLI2(S236);:23.17818429152722\n",
      "IGF2R(S2409);:20.230175341507017\n",
      "SPP1(S219);:19.339562999115902\n",
      "EPN2(S195);:14.837722069198291\n",
      "CCDC86(S58);:14.098201273561386\n",
      "EML3(S177);:13.590920264559813\n",
      "------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#IC50 prediction\n",
    "y_pred = model2.predict(X_test[idx:(idx+1)])  \n",
    "print('------------------------------------------------')\n",
    "print(f'True IC50 value: {y_train[idx:(idx+1)].values[0]}\\nPredicted IC50 value: {y_pred[0][0]}')\n",
    "\n",
    "#explainer instance\n",
    "explanation = ig2.explain(X_train[idx:(idx+1)].values,\n",
    "                         baselines=None,\n",
    "                         target=None)\n",
    "\n",
    "#print the top features for this explanation \n",
    "attributions = explanation.attributions\n",
    "#this function outputs the top x number of features and their scores for a model\n",
    "print('------------------------------------------------')\n",
    "NN_final_names, NN_final_scores = rfrFeatures(attributions[0][0], X_main = X_train, topX = num_feats, N = num_feats)\n",
    "for i in range(10):\n",
    "    print(f'{NN_final_names[i]}:{NN_final_scores[i]}')\n",
    "print('------------------------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3416eede-a410-412a-bb18-52b1caafcfe6",
   "metadata": {},
   "source": [
    "Late integration DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "6ac12d43-5a56-4f70-a946-d02e0ac1ae34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 108ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "It looks like you are passing a model with a scalar output and target is set to `None`.If your model is a regression model this will produce correct attributions. If your model is a classification model, targets for each datapoint must be defined. Not defining the target may lead to incorrect values for the attributions.Targets can be either the true classes or the classes predicted by the model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------\n",
      "True IC50 value: 0.948509\n",
      "Predicted IC50 value: -0.5204095244407654\n",
      "------------------------------------------------\n",
      "CDK1(S233);:0.016885131329390082\n",
      "MAP7(S365);:0.010485951762391822\n",
      "NOSIP(S107);:0.010390026537665449\n",
      "SPP1(S215);:0.008735702721325291\n",
      "MYCBP2(S2774);:0.00822409793527249\n",
      "RAP1GAP(T486);:0.006853564907783864\n",
      "RAP1GAP(S484);:0.0060732027869332805\n",
      "MSH6(S43);:0.0053999583465289165\n",
      "RRP8(S51);RRP8(S53);:0.005390667881927584\n",
      "KAT6B(S1597);KAT6B(S1598);:0.004900299172853961\n",
      "------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#IC50 prediction\n",
    "y_pred = model3.predict([xo_test[idx:(idx+1)], xd_test[idx:(idx+1)]])  \n",
    "print('------------------------------------------------')\n",
    "print(f'True IC50 value: {y_train[idx:(idx+1)].values[0]}\\nPredicted IC50 value: {y_pred[0][0]}')\n",
    "\n",
    "#explainer instance\n",
    "explanation = ig3.explain([xo_train[idx:(idx+1)].values, np.array(xd_train[idx:(idx+1)])],\n",
    "                         baselines=None,\n",
    "                         target=None)\n",
    "\n",
    "#print the top features for this explanation\n",
    "attributions = explanation.attributions\n",
    "#this function outputs the top x number of features and their scores for a model\n",
    "print('------------------------------------------------')\n",
    "NN_final_names, NN_final_scores = rfrFeatures(attributions[0][0], X_main = X_train, topX = num_feats, N = num_feats)\n",
    "for i in range(10):\n",
    "    print(f'{NN_final_names[i]}:{NN_final_scores[i]}')\n",
    "print('------------------------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4a34433-aa69-40dd-9a74-d6edca38f8e9",
   "metadata": {},
   "source": [
    "Late integration CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "8da0710e-4a4d-48e4-8ccf-d3ce63cff15d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 166ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "It looks like you are passing a model with a scalar output and target is set to `None`.If your model is a regression model this will produce correct attributions. If your model is a classification model, targets for each datapoint must be defined. Not defining the target may lead to incorrect values for the attributions.Targets can be either the true classes or the classes predicted by the model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------\n",
      "True IC50 value: 0.948509\n",
      "Predicted IC50 value: -0.7935070991516113\n",
      "------------------------------------------------\n",
      "DDX10(S793);:7.974487250216398\n",
      "JUN(S63);:7.917300081603442\n",
      "KAT6B(S1597);KAT6B(S1598);:5.064132690122426\n",
      "KAT6B(S1581);:3.9622710298443082\n",
      "CDK1(S233);:3.486010492916559\n",
      "IKZF1(S442);:3.157174011427876\n",
      "MACF1(S5718);:3.080785043651632\n",
      "HOXA10(S282);:1.630120018788577\n",
      "CCDC86(S37);:1.512939385380748\n",
      "CRTAP(S250);:1.38290933649561\n",
      "------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#IC50 prediction\n",
    "y_pred = model4.predict([xo_test[idx:(idx+1)], xd_test[idx:(idx+1)]]) \n",
    "print('------------------------------------------------')\n",
    "print(f'True IC50 value: {y_train[idx:(idx+1)].values[0]}\\nPredicted IC50 value: {y_pred[0][0]}')\n",
    "\n",
    "#explainer instance\n",
    "explanation = ig4.explain([xo_train[idx:(idx+1)].values, np.array(xd_train[idx:(idx+1)])],\n",
    "                         baselines=None,\n",
    "                         target=None)\n",
    "#print the top features for this explanation\n",
    "attributions = explanation.attributions\n",
    "#this function outputs the top x number of features and their scores for a model\n",
    "print('------------------------------------------------')\n",
    "NN_final_names, NN_final_scores = rfrFeatures(attributions[0][0], X_main = X_train, topX = num_feats, N = num_feats)\n",
    "for i in range(10):\n",
    "    print(f'{NN_final_names[i]}:{NN_final_scores[i]}')\n",
    "print('------------------------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26cc5e65-3e2d-4b2e-9084-6fea8a600b60",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Performance plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4afb9c3c-7133-4e13-b10c-a624cf7c83b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics1 = pd.read_csv('plots/phospho/NormalisedX_CNN_metrics_randomFeats.csv')\n",
    "metrics2 = pd.read_csv('plots/phospho/NormalisedX_CNN_metrics_L1000.csv')\n",
    "#xgb_metrics = pd.read_csv('plots/phospho/xgb_metrics.csv')\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n",
    "\n",
    "# Plot 1\n",
    "axes[0].plot(metrics1['rs'].astype(str), metrics1['mm_r2'], linestyle='-', marker='.', color='orange', linewidth=1)\n",
    "axes[0].plot(metrics1['rs'].astype(str), metrics1['mixed_set_r2'], linestyle='-', marker='.', color='yellow', linewidth=1)\n",
    "axes[0].plot(metrics1['rs'].astype(str), metrics1['model_r2'], linestyle='-', marker='.', color='blue', linewidth=1)\n",
    "axes[0].set_title('Random Features')\n",
    "\n",
    "# Plot 2\n",
    "axes[1].plot(metrics2['rs'].astype(str), metrics2['mm_r2'], linestyle='-', marker='.', color='orange', linewidth=1)\n",
    "axes[1].plot(metrics2['rs'].astype(str), metrics2['mixed_set_r2'], linestyle='-', marker='.', color='yellow', linewidth=1)\n",
    "axes[1].plot(metrics2['rs'].astype(str), metrics2['model_r2'], linestyle='-', marker='.', color='blue', linewidth=1)\n",
    "axes[1].set_title('L1000')\n",
    "\n",
    "# Plot 3\n",
    "#axes[2].plot(shap_df_features['rs'].astype(str), shap_df_features['mm_r2'], linestyle='-', marker='.', color='orange', linewidth=1)\n",
    "#axes[2].plot(shap_df_features['rs'].astype(str), shap_df_features['model_r2'], linestyle='-', marker='.', color='blue', linewidth=1)\n",
    "#axes[2].set_title('Features Filtered')\n",
    "\n",
    "# Adjust spacing between subplots\n",
    "plt.tight_layout()\n",
    "\n",
    "# Display the plots\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74510e52-89b2-4a46-9a4a-ddcb62cbd075",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics1 = pd.read_csv('plots/phospho/NormalisedX_CNN_metrics_randomFeats.csv')\n",
    "metrics2 = pd.read_csv('plots/phospho/NormalisedX_CNN_metrics_L1000.csv')\n",
    "#xgb_metrics = pd.read_csv('plots/phospho/xgb_metrics.csv')\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n",
    "\n",
    "# Plot 1\n",
    "axes[0].plot(metrics1['rs'].astype(str), metrics1['model_r2'], linestyle='-', marker='.', color='yellow', linewidth=1)\n",
    "axes[0].plot(metrics2['rs'].astype(str), metrics2['model_r2'], linestyle='-', marker='.', color='blue', linewidth=1)\n",
    "axes[0].set_title('Random Features')\n",
    "\n",
    "# Adjust spacing between subplots\n",
    "plt.tight_layout()\n",
    "\n",
    "# Display the plots\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c34c0e84-c072-49f7-8255-706985bde995",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}