TMP_RandomForest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import functools\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas\n",
    "from datetime import datetime\n",
    "from glob import glob\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, balanced_accuracy_score\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.utils._joblib import Parallel, delayed, effective_n_jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fileMap = {}\n",
    "for i in glob(\"data/v8-feature-matrices/*.tsv\"):\n",
    "    if \"5FOLD\" not in i:\n",
    "        name = os.path.basename(i).split(\"_\")[0]\n",
    "        fileMap[name] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "crossMap = {}\n",
    "for i in glob(\"data/v8-cv-matrices/*.tsv\"):\n",
    "    if \"5FOLD\" in i:\n",
    "        name = os.path.basename(i).split(\"_\")[0]\n",
    "        crossMap[name] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "featureSelection = {}\n",
    "\n",
    "df = pandas.read_csv(\"data/featuresets_v8.tsv\", sep=\"\\t\")\n",
    "df[\"TCGA_Projects\"] = df[\"TCGA_Projects\"].apply(json.loads)\n",
    "df[\"Features\"] = df[\"Features\"].apply(json.loads)\n",
    "df = pandas.DataFrame({'Feature_Set_ID': np.repeat(df.Feature_Set_ID.values, df.TCGA_Projects.str.len()),\n",
    "                       'TCGA_Project': np.concatenate(df.TCGA_Projects.values),\n",
    "                       'Features': np.repeat(df.Features.values, df.TCGA_Projects.str.len())})\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    cancer = row['TCGA_Project']\n",
    "    if cancer not in featureSelection:\n",
    "        featureSelection[cancer] = {}\n",
    "    fset = row['Feature_Set_ID']\n",
    "    if fset not in featureSelection[cancer]:\n",
    "        featureSelection[cancer][fset] = row['Features']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dropna(v):\n",
    "    return list(filter(lambda x:x==x, v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def runRandomForest(cancer, fset, seed = 42, n_estimators = 500):\n",
    "    model_id = \"%s|%s|%s|c\" % (\"%s:RandomForest(n_estimators=%s, random_state=%s)\" % (cancer, n_estimators, seed), \n",
    "                               fset, \n",
    "                               datetime.now(tz=None).isoformat())\n",
    "    print(\"Running %s\" % (model_id))\n",
    "    \n",
    "    matrix = pandas.read_csv(fileMap[cancer], sep=\"\\t\", index_col=0)\n",
    "    \n",
    "    try:\n",
    "        X = matrix[dropna(featureSelection[cancer][fset])]\n",
    "        y = matrix.Labels\n",
    "    except Exception as e:\n",
    "        print(\"Exception:\", e)\n",
    "        return pandas.DataFrame()\n",
    "    \n",
    "    if len(X.columns) == 0:\n",
    "        return pandas.DataFrame()\n",
    "    \n",
    "    crossf = pandas.read_csv(crossMap[cancer], delimiter=\"\\t\", index_col=0)\n",
    "    repeat_folds = list(crossf.columns)\n",
    "    repeat_folds.pop(0) # remove entry called \"Labels\"\n",
    "\n",
    "    preds = pandas.DataFrame()\n",
    "    for ct in repeat_folds:\n",
    "        X_test = X[crossf[ct] == 1]\n",
    "        y_test = y[crossf[ct] == 1]\n",
    "        X_train = X[crossf[ct] == 0]\n",
    "        y_train = y[crossf[ct] == 0]\n",
    "\n",
    "        clf = RandomForestClassifier(n_estimators=500, random_state = seed)\n",
    "        try:\n",
    "            clf.fit(X_train, y_train)\n",
    "        except Exception as e:\n",
    "            print(\"Exception:\", e)\n",
    "            continue\n",
    "        train_pred = clf.predict(X_train)\n",
    "        test_pred = clf.predict(X_test)\n",
    "        test = pandas.DataFrame({\"Sample_ID\": X_test.index, \n",
    "                                 \"Repeat\": int(ct.split(\":\")[0][1:]),\n",
    "                                 \"Fold\": int(ct.split(\":\")[1][1:]) , \n",
    "                                 \"Test\": 1, \n",
    "                                 \"Label\": [\"%s:%s\" % (cancer, y) for y in y_test], \n",
    "                                 model_id: [\"%s:%s\" % (cancer, y) for y in test_pred]})\n",
    "        train = pandas.DataFrame({\"Sample_ID\": X_train.index, \n",
    "                                  \"Repeat\": int(ct.split(\":\")[0][1:]),\n",
    "                                  \"Fold\": int(ct.split(\":\")[1][1:]) , \n",
    "                                  \"Test\": 0, \n",
    "                                  \"Label\": [\"%s:%s\" % (cancer, y) for y in y_train], \n",
    "                                  model_id: [\"%s:%s\" % (cancer, y) for y in train_pred]})\n",
    "        preds = preds.append(test).append(train)\n",
    "        \n",
    "    return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running jobs for: LIHCCHOL\n",
      "running jobs for: CESC\n",
      "running jobs for: LGGGBM\n"
     ]
    }
   ],
   "source": [
    "sFunc = delayed(runRandomForest)\n",
    "parallel = Parallel(n_jobs=10)\n",
    "for cancer in featureSelection.keys():\n",
    "    if cancer in results:\n",
    "        continue\n",
    "    f = []\n",
    "    for fset in featureSelection[cancer].keys():\n",
    "        f.append( sFunc(cancer, fset) )\n",
    "    print(\"running jobs for: %s\" % cancer)\n",
    "    try:\n",
    "        o = parallel(f)\n",
    "    except:\n",
    "        o = Prallel(n_jobs=10)(f)\n",
    "    results[cancer] = o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# writing prediction output files\n",
    "for c in results.keys():\n",
    "    cpred = functools.reduce(pandas.merge, [df for df in results[c] if df.shape[0] > 0])\n",
    "    cpred.to_csv(\"./outputs/%s_randomforest_v8_struck.tsv\" % c, index = False, sep = \"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "python3"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "nteract": {
   "version": "0.15.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}