GateCellTypes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load libraries\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import copy\n",
    "import seaborn as sns\n",
    "import importlib\n",
    "from matplotlib import cm\n",
    "import matplotlib as mpl\n",
    "mpl.rc('figure', max_open_warning = 0)\n",
    "mpl.rcParams['pdf.fonttype'] = 42\n",
    "mpl.rcParams['ps.fonttype'] = 42\n",
    "mpl.rcParams['mathtext.fontset'] = 'custom'\n",
    "mpl.rcParams['mathtext.it'] = 'Arial:italic'\n",
    "mpl.rcParams['mathtext.rm'] = 'Arial'\n",
    "codedir = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load cmif libraries\n",
    "#os.chdir('/home/groups/graylab_share/OMERO.rdsStore/engje/Data/cmIF')\n",
    "from mplex_image import visualize as viz, process, preprocess, gating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(codedir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notes\n",
    "\n",
    "use CD45 to gate immune (CD3 more artifact)\n",
    "\n",
    "update 20200402: add SMT-Bx2-5 and HTA-33, simplified gating."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set location of files\n",
    "#load data\n",
    "rootdir = f'{codedir}/paper_data'\n",
    "# go to location of files\n",
    "os.chdir(rootdir)\n",
    "preprocess.cmif_mkdir(['GatingPlots'])\n",
    "#os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3 define samples to work with/ image combos\n",
    "ls_sample = ['20210402_SMT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = pd.DataFrame()\n",
    "for s_sample in ls_sample:\n",
    "    df_data = df_data.append(pd.read_csv(f'{s_sample}_ManualPositive.csv',index_col=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_rename = {'CD4':'CD4_Ring','CD8':'CD8_Ring',\n",
    "           #'HER2':'HER2_Ring','ER':'ER_Nuclei'\n",
    "           }\n",
    "df_data = df_data.rename(d_rename, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specify Gating Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#parameters\n",
    "\n",
    "# cell types\n",
    "ls_endothelial = ['CD31']\n",
    "ls_immune = ['CD45','CD68'] \n",
    "ls_tumor = ['CK7','CK19','Ecad'] \n",
    "ls_prolif = ['Ki67']\n",
    "\n",
    "#tcell/myeloid\n",
    "s_tcell = 'CD45' \n",
    "s_bcell = 'CD20'\n",
    "s_myeloid = 'CD68'\n",
    "ls_immune_functional = ['PD1','CD44','prolif'] # not in dataset: 'FoxP3_Nuclei','GRNZB_Nuclei',\n",
    "\n",
    "#luminal/basal/mesenchymal\n",
    "ls_luminal = ['CK19','CK7'] # not in dataset 'CK8_Ring'\n",
    "ls_basal = ['CK5','CK14'] \n",
    "ls_mes = ['CD44', 'Vim'] \n",
    "ls_tumor_plus = ['Ecad'] + ['Lum']\n",
    "ls_stromal_function =  ['Vim','aSMA','PDPN']\n",
    "ls_tumor_prolif = ['PCNA','Ki67','pHH3']  \n",
    "\n",
    "#index of cell line samples (i.e. 100% tumor)\n",
    "ls_cellline_index = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#custom gating\n",
    "df_data = gating.main_celltypes(df_data,ls_endothelial,ls_immune,ls_tumor,ls_cellline_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#add normal liver\n",
    "df_data.loc[(~df_data.loc[:,ls_luminal].any(axis=1) & df_data.loc[:,'Ecad'] & df_data.loc[:,'tumor']),'celltype'] = 'epithelial'\n",
    "df_data.loc[df_data.celltype == 'epithelial','tumor'] = False\n",
    "df_data.loc[df_data.celltype == 'epithelial','epithelial'] = True\n",
    "df_data.loc[df_data.celltype != 'epithelial','epithelial'] = False\n",
    "df_data.epithelial = df_data.epithelial.astype('bool')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(gating)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform Gating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "#simple gating\n",
    "df_data = gating.proliferation(df_data,ls_prolif)\n",
    "df_data = gating.immune_types(df_data,s_myeloid,s_bcell,s_tcell)\n",
    "df_data = gating.cell_prolif(df_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "#cutom gating (skip)\n",
    "'''\n",
    "df_data = gating.immune_functional(df_data,ls_immune_functional)\n",
    "df_data = gating.diff_hr_state(df_data,ls_luminal,ls_basal,ls_mes)\n",
    "df_data = gating.celltype_gates(df_data,ls_tumor_prolif,s_new_name='TumorProlif',s_celltype='tumor')\n",
    "#df_data = gating.celltype_gates(df_data,ls_tumor_plus,s_new_name='TumorDiffPlus',s_celltype='tumor')\n",
    "df_data = gating.celltype_gates(df_data,ls_stromal_function,s_new_name='StromalType',s_celltype='stromal')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = gating.non_tumor(df_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output Gating Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check\n",
    "ls_drop = ['ColI', 'ColIV', 'CD20', 'CD3', 'CD44', 'CK14',\n",
    "       'CK5', 'ER', 'HER2', 'LamAC', 'PCNA', 'PD1', 'pHH3']\n",
    "df_data.loc[:,df_data.dtypes=='object'].drop(ls_drop,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop extra colums\n",
    "df_gate = df_data.loc[:,df_data.dtypes!='bool'].drop(ls_drop,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#handcrafted stromal populations (skip)\n",
    "'''\n",
    "d_rename_stroma = {'stromal_Vim_aSMA':'myofibroblast', 'stromal_aSMA':'myofibroblast', 'stromal___':'stromal', 'stromal_Vim':'fibroblast',\n",
    "       'stromal_PDPN_Vim_aSMA':'myofibroblast', 'stromal_PDPN_Vim':'fibroblast', 'stromal_PDPN':'lymphatic',\n",
    "       'stromal_PDPN_aSMA':'myofibroblast'}\n",
    "df_gate.NonTumor = df_gate.NonTumor.replace(d_rename_stroma)\n",
    "df_gate['FinalCell'] = df_gate.NonTumor.fillna(df_gate.CellProlif).fillna(df_gate.celltype)\n",
    "df_gate.FinalCell = df_gate.FinalCell.replace({'tumor_nonprolif':'tumor','liver_nonprolif':'liver','liver_prolif':'liver'})\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_gate.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s_out = '20210402_SMT'\n",
    "if not os.path.exists(f'{s_out}_GatedPositiveCellNames.csv'):\n",
    "    print('saving new csv')\n",
    "    df_gate.to_csv(f'{s_out}_GatedPositiveCellNames.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importlib.reload(viz)\n",
    "s_out = '20210402_SMT'\n",
    "f'{s_out}_GatedPositiveCellNames.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = pd.read_csv(f'{s_out}_GatedPositiveCellNames.csv',index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_data['Stromal'] = df_data.StromalType.replace(d_rename_stroma)\n",
    "#df_data['NonTumor'] = df_data.NonTumor.replace(d_rename_stroma)\n",
    "#df_data['NonTumorFunc'] = df_data.NonTumorFunc.replace(d_rename_stroma)\n",
    "#handcrafted stromal populations\n",
    "#d_rename_stroma = {'stromal_Vim_aSMA':'myofibroblast', 'stromal_aSMA':'myofibroblast', 'stromal___':'stromal', 'stromal_Vim':'fibroblast',\n",
    "#       'stromal_PDPN_Vim_aSMA':'myofibroblast', 'stromal_PDPN_Vim':'fibroblast', 'stromal_PDPN':'lymphatic',\n",
    "#       'stromal_PDPN_aSMA':'myofibroblast'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(df_data.columns == 'FinalCell').any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combined cell type (run once)\n",
    "if not (df_data.columns == 'FinalCell').any():\n",
    "    df_data.loc[df_data.celltype == 'tumor','FinalCell'] = df_data.loc[df_data.celltype == 'tumor','CellProlif']\n",
    "    df_data.loc[df_data.celltype != 'tumor','FinalCell'] = df_data.loc[df_data.celltype != 'tumor','celltype']\n",
    "    df_data.loc[df_data.celltype == 'immune','FinalCell'] = df_data.loc[df_data.celltype == 'immune','ImmuneType']\n",
    "\n",
    "#df_data.FinalCell.unique()\n",
    "#df_data.to_csv(f'{s_out}_GatedPositiveCellNames.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls_drop = df_data.loc[((df_data.index.str.contains('HTA')) & (df_data.FinalCell=='epithelial'))].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get rid epithelial\n",
    "# except HTAN\n",
    "df_data['FinalCell'] = df_data.FinalCell.replace({'epithelial':'stromal'})\n",
    "df_data = df_data.drop(ls_drop)\n",
    "df_data['countme'] = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "s_grouper='slide_scene'\n",
    "\n",
    "#calculate proportions\n",
    "for s_cell in df_data.columns[(df_data.dtypes=='object') & ~(df_data.columns.isin([s_grouper]))].tolist():\n",
    "    df_prop = viz.prop_positive(df_data,s_cell=s_cell,s_grouper=s_grouper)\n",
    "    # make annotations\n",
    "    df_annot=pd.DataFrame(data={'ID': df_prop.index.tolist()},index=df_prop.index)\n",
    "    lut = dict(zip(sorted(df_annot.ID.unique()),cm.tab10.colors))\n",
    "    g, df_plot_less = viz.prop_clustermap(df_prop,df_annot,i_thresh =.01,lut=lut)\n",
    "    g.savefig(f'./GatingPlots/{s_cell}_clustermap.png',dpi=150)\n",
    "    plt.close()\n",
    "    fig = viz.prop_barplot(df_plot_less,s_cell,colormap=\"Spectral\")\n",
    "    fig.savefig(f'./GatingPlots/{s_cell}_bar.png',dpi=200)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#group by tissue\n",
    "df_data['slide_scene'] = [item.split('_')[0] for item in df_data.slide_scene]\n",
    "df_data_select = df_data.loc[~df_data.slide_scene.isin(['HTA-33_scene001','SMTBx1-16_scene001']),:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#by tissue\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "s_grouper='slide_scene'\n",
    "mpl.rcParams['pdf.fonttype'] = 42\n",
    "mpl.rcParams['ps.fonttype'] = 42\n",
    "\n",
    "#calculate proportions\n",
    "for s_cell in df_data.columns[(df_data.dtypes=='object') & ~(df_data.columns.isin([s_grouper]))].tolist():\n",
    "    df_prop = viz.prop_positive(df_data_select,s_cell=s_cell,s_grouper=s_grouper)\n",
    "    # make annotations\n",
    "    df_prop.to_csv(f'ManualGating_SMT_proportions_{s_cell}.csv')\n",
    "    df_annot=pd.DataFrame(data={'ID': df_prop.index.tolist()},index=df_prop.index)\n",
    "    lut = dict(zip(sorted(df_annot.ID.unique()),cm.tab10.colors))\n",
    "    g, df_plot_less = viz.prop_clustermap(df_prop,df_annot,i_thresh =.001,lut=lut)\n",
    "    g.savefig(f'./GatingPlots/{s_cell}_clustermap_tissue.pdf',dpi=150)\n",
    "    plt.close()\n",
    "    if df_plot_less.shape[1] < 8:\n",
    "        cmap = \"Spectral\"\n",
    "    elif df_plot_less.shape[1] < 11:\n",
    "        cmap = \"Paired\"\n",
    "    else:\n",
    "        cmap = \"tab20\"\n",
    "    fig = viz.prop_barplot(df_plot_less,s_cell,colormap=cmap)\n",
    "    fig.savefig(f'./GatingPlots/{s_cell}_bar_tissue.pdf',dpi=200)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s_date = '20210402'\n",
    "d_crop = {'SMTBx2-5_scene001': (2000,9000),\n",
    "    'SMTBx3_scene004': (20000,16000),\n",
    "    'HTA-33_scene002': (3271, 607),\n",
    "    'SMTBx1-16_scene003': (2440,220),\n",
    "        }\n",
    "df_result = pd.DataFrame()\n",
    "for s_tissue, tu_crop in d_crop.items():\n",
    "    df_scene = df_data.loc[df_data.index.str.contains(s_tissue)]\n",
    "    ls_index = df_scene.loc[((df_scene.DAPI_X > tu_crop[0]) & (df_scene.DAPI_X < tu_crop[0]+2500)) & (df_scene.DAPI_Y > tu_crop[1]) & (df_scene.DAPI_Y < tu_crop[1]+2500)].index\n",
    "    df_result = df_result.append(df_data.loc[ls_index])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#by tissue\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "s_grouper='slide_scene'\n",
    "mpl.rcParams['pdf.fonttype'] = 42\n",
    "mpl.rcParams['ps.fonttype'] = 42\n",
    "d_rename = {'HTA-33':'Bx4', 'SMTBx1-16':'Bx1', 'SMTBx2-5':'Bx2', 'SMTBx3':'Bx3'}\n",
    "\n",
    "#calculate proportions\n",
    "for s_cell in df_data.columns[(df_data.dtypes=='object') & ~(df_data.columns.isin([s_grouper]))].tolist():\n",
    "    df_prop = viz.prop_positive(df_result,s_cell=s_cell,s_grouper=s_grouper)\n",
    "    # make annotations\n",
    "    #df_prop.to_csv(f'ManualGating_SMT101_proportions_{s_cell}.csv')\n",
    "    df_annot=pd.DataFrame(data={'ID': df_prop.index.tolist()},index=df_prop.index)\n",
    "    lut = dict(zip(sorted(df_annot.ID.unique()),cm.tab10.colors))\n",
    "    g, df_plot_less = viz.prop_clustermap(df_prop,df_annot,i_thresh =.001,lut=lut)\n",
    "    g.savefig(f'./GatingPlots/{s_cell}_clustermap_tissue3.pdf',dpi=150)\n",
    "    plt.close()\n",
    "    if df_plot_less.shape[1] < 8:\n",
    "        cmap = \"Spectral\"\n",
    "    elif df_plot_less.shape[1] < 11:\n",
    "        cmap = \"Paired\"\n",
    "    else:\n",
    "        cmap = \"tab20\"\n",
    "    fig = viz.prop_barplot(df_plot_less.rename(d_rename),s_cell,colormap=cmap)\n",
    "    fig.set_size_inches(4.5, 2.3)\n",
    "    ax_list = fig.axes\n",
    "    ax_list[0].set_ylabel('')\n",
    "    ax_list[0].set_xlabel('Fraction of Cells')\n",
    "    ax_list[0].set_title('')\n",
    "    fig.suptitle('Gating Composition: Biopsies',x=0.5,y=0.9,fontsize=14)\n",
    "    plt.tight_layout()\n",
    "    fig.savefig(f'./GatingPlots/{s_cell}_bar_tissue3.png',dpi=200)\n",
    "    #fig.savefig(f'./{s_date}/{s_cell}_bar_tissue3.pdf',dpi=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3.9.5",
   "language": "python",
   "name": "python3.9.5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}