metrics_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrics evaluation notebook\n",
    "\n",
    "_Alex Malz (NYU)_\n",
    "_Renee Hlozek (Toronto)_\n",
    "_Rahul Biswas (Stockholm University)_\n",
    "_Tarek Alam (UCL)_\n",
    "_Rafael Martinez-Galarza (Harvard)_\n",
    "_Anita Bahmanyar (Toronto)_\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import string\n",
    "# import itertools\n",
    "# import random\n",
    "# import os\n",
    "# import csv\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pycm import ConfusionMatrix\n",
    "\n",
    "import proclam\n",
    "from proclam import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "mpl.use('Agg')\n",
    "mpl.rcParams['text.usetex'] = False\n",
    "mpl.rcParams['mathtext.rm'] = 'serif'\n",
    "mpl.rcParams['font.family'] = 'serif'\n",
    "mpl.rcParams['font.serif'] = 'Times New Roman'\n",
    "mpl.rcParams['axes.titlesize'] = 16\n",
    "mpl.rcParams['axes.labelsize'] = 14\n",
    "mpl.rcParams['savefig.dpi'] = 250\n",
    "mpl.rcParams['savefig.format'] = 'pdf'\n",
    "mpl.rcParams['savefig.bbox'] = 'tight'\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook defines the metrics (and corresponding truth tables) to be run via an input file and to produce plots based on the output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metricslist = ['Brier', 'LogLoss']\n",
    "colors = ['b', 'r']\n",
    "dirname = 'examples/'\n",
    "markerlist = ['o', 's', '*']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read in data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading in the data from the Mystery Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mystery = {}\n",
    "mystery['label'] = 'Unknown'\n",
    "mystery['names'] = ['RandomForest', 'KNeighbors', 'MLPNeuralNet']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading in the data from SNPhotCC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "snphotcc = {}\n",
    "snphotcc['label'] = 'SNPhotCC'\n",
    "prefixes = ['Templates', 'Wavelets']\n",
    "suffixes = ['BoostForest', 'KNN', 'NB', 'NeuralNetwork', 'SVM']\n",
    "snphotcc['names'] = []\n",
    "for prefix in prefixes:\n",
    "    for suffix in suffixes:\n",
    "        snphotcc['names'].append(prefix+suffix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading in the data from ProClaM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plasticc = {}\n",
    "plasticc['label'] = 'ProClaM'\n",
    "plasticc['names'] = ['Idealized', 'Guess', 'Tunnel', 'Broadbrush', 'Cruise', 'SubsumedTo', 'SubsumedFrom']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# old_snphotcc_names = []\n",
    "# for prefix in ['templates_', 'wavelets_']:\n",
    "#     for suffix in ['boost_forest', 'knn', 'nb', 'neural_network', 'svm']:\n",
    "#         old_snphotcc_names.append(prefix+suffix+'.dat')\n",
    "\n",
    "# for i in range(len(snphotcc_names)):\n",
    "#     name = old_snphotcc_names[i]\n",
    "#     fileloc = dirname+'classifications/'+name\n",
    "#     snphotcc_info = pd.read_csv(fileloc, sep=' ')\n",
    "#     full = snphotcc_info.set_index('Object').join(truth_snphotcc.set_index('Object'))\n",
    "#     name = snphotcc_names[i]\n",
    "    \n",
    "#     truth = full['Type'] - 1\n",
    "#     snphotcc_truth_table = proclam.metrics.util.det_to_prob(truth)\n",
    "#     fileloc = 'examples/'+name+'/truth_table_'+name+'.csv'\n",
    "#     with open(fileloc, 'wb') as truth_place:\n",
    "#         np.savetxt(fileloc, snphotcc_truth_table, delimiter=' ')\n",
    "    \n",
    "#     probs = full[['1', '2', '3']]\n",
    "#     fileloc = 'examples/'+name+'/predicted_prob_'+name+'.csv'\n",
    "#     probs.to_csv(fileloc, sep=' ', index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# more_names = snphotcc_names\n",
    "# more_classifications = ['%s/predicted_prob_%s.csv'%(name,name) for name in snphotcc_names]\n",
    "# more_truth_tables = ['%s/truth_table_%s.csv'%(name,name) for name in snphotcc_names]\n",
    "# more_class_pairs = zip(more_classifications, more_truth_tables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_class_pairs(data_info_dict):\n",
    "    return zip(data_info_dict['classifications'], data_info_dict['truth_tables'])\n",
    "\n",
    "def make_file_locs(data_info_dict):\n",
    "    names = data_info_dict['names']\n",
    "    data_info_dict['dirname'] = dirname + data_info_dict['label'] + '/'\n",
    "    data_info_dict['classifications'] = ['%s/predicted_prob_%s.csv'%(name, name) for name in names]\n",
    "    data_info_dict['truth_tables'] = ['%s/truth_table_%s.csv'%(name, name) for name in names]\n",
    "#     print(data_info_dict)\n",
    "    return data_info_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in [mystery, snphotcc, plasticc]:\n",
    "    dataset = make_file_locs(dataset)\n",
    "    dataset['class_pairs'] = make_class_pairs(dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Make the plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_cm(probs, truth, name, loc=''):\n",
    "    cm = proclam.metrics.util.prob_to_cm(probs, truth)\n",
    "    plt.matshow(cm.T, vmin=0., vmax=1.)\n",
    "# plt.xticks(range(max(truth)+1), names)\n",
    "# plt.yticks(range(max(truth)+1), names)\n",
    "    plt.xlabel('predicted class')\n",
    "    plt.ylabel('true class')\n",
    "    plt.colorbar()\n",
    "    plt.title(name)\n",
    "    plt.savefig(loc+name+'_cm.png')\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_class_pairs(pair, dataset, cc):#loc='', title=''):\n",
    "    loc=dataset['dirname']\n",
    "    title=dataset['label']+' '+dataset['names'][cc]\n",
    "    clfile = pair[0]\n",
    "    truthfile = pair[1]\n",
    "    prob_mat = pd.read_csv(loc+clfile, delim_whitespace=True).values\n",
    "    nobj = np.shape(prob_mat)[0]\n",
    "    nclass = np.shape(prob_mat)[1]\n",
    "    truth_values = pd.read_csv(loc+truthfile, delim_whitespace=True).values\n",
    "    nobj_truth = np.shape(truth_values)[0]\n",
    "    nclass_truth = np.shape(truth_values)[1]\n",
    "    tvec = np.where(truth_values==1)[1]\n",
    "#     if nclass_truth!= nclass:\n",
    "#         print('Truth table of size %i x %i and prob matrix of size %i x %i do not match up in size'%(nobj,nclass,nobj_truth,nclass_truth))\n",
    "#     else:\n",
    "#         print('Considering classifications with %i classes'%nclass)\n",
    "    pmat = prob_mat\n",
    "    plot_cm(pmat, tvec, title, loc=loc+dataset['names'][cc]+'/')\n",
    "    return pmat, tvec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_patch_spines_invisible(ax):\n",
    "    ax.set_frame_on(True)\n",
    "    ax.patch.set_visible(False)\n",
    "    for sp in ax.spines.values():\n",
    "        sp.set_visible(False)\n",
    "        \n",
    "def per_metric_helper(ax, n, data, metric_names, codes, shapes, colors):\n",
    "    plot_n = n+1\n",
    "    in_x = np.arange(len(codes))\n",
    "    ax_n = ax\n",
    "    n_factor = 0.1 * (plot_n - 2)\n",
    "    if plot_n>1:\n",
    "        ax_n = ax.twinx()\n",
    "        rot_ang = 270\n",
    "        label_space = 15.\n",
    "    else:\n",
    "        rot_ang = 90\n",
    "        label_space = 0.\n",
    "    if plot_n>2:\n",
    "        ax_n.spines[\"right\"].set_position((\"axes\", 1. + 0.1 * (plot_n-1)))\n",
    "        make_patch_spines_invisible(ax_n)\n",
    "        ax_n.spines[\"right\"].set_visible(True)\n",
    "    handle = ax_n.scatter(in_x+n_factor*np.ones_like(data[n]), data[n], marker=shapes[n], s=10, color=colors[n], label=metric_names[n])\n",
    "    ax_n.set_ylabel(metric_names[n], rotation=rot_ang, fontsize=14, labelpad=label_space)\n",
    "#     ax_n.set_ylim(0.9 * min(data[n]), 1.1 * max(data[n]))\n",
    "    return(ax, ax_n, handle)\n",
    "\n",
    "def metric_plot(dataset, metric_names, shapes, colors):\n",
    "    codes = dataset['names']\n",
    "    data = dataset['results']\n",
    "    title = dataset['label']\n",
    "    fileloc = dataset['dirname']+dataset['label']+'_results.png'\n",
    "    xs = np.arange(len(codes))\n",
    "    fig, ax = plt.subplots()\n",
    "    fig.subplots_adjust(right=1.)\n",
    "    handles = []\n",
    "    for n in range(len(metric_names)):\n",
    "        (ax, ax_n, handle) = per_metric_helper(ax, n, data, metric_names, codes, shapes, colors)\n",
    "        handles.append(handle)\n",
    "    plt.xticks(xs, codes)\n",
    "    for tick in ax.get_xticklabels():\n",
    "        tick.set_rotation(90)\n",
    "    plt.xlabel('Classifiers', fontsize=14)\n",
    "    plt.legend(handles, metric_names)\n",
    "    plt.suptitle(title)\n",
    "    plt.savefig(fileloc)\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "make a plot of classifier on the x-axis and metric score on the y-axis\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in [mystery, snphotcc, plasticc]:\n",
    "    data = np.empty((len(metricslist), len(dataset['names'])))\n",
    "    for cc, pair in enumerate(dataset['class_pairs']):\n",
    "        probm, truthv = read_class_pairs(pair, dataset, cc)#loc=dataset['dirname'], title=dataset['label']+' '+dataset['names'][cc])\n",
    "#         plot_cm(probm, truthv, str(cc), loc='./sandbox/')\n",
    "        det = proclam.metrics.util.prob_to_det(probm)\n",
    "        cm = proclam.metrics.util.prob_to_cm(probm, truthv, per_class_norm=False, vb=False)\n",
    "        rates = proclam.metrics.util.cm_to_rate(cm, vb=True)\n",
    "#         print(rates)\n",
    "        compare = ConfusionMatrix(truthv, det)\n",
    "        printout = proclam.metrics.util.RateMatrix(compare.TPR, compare.FPR, compare.FNR, compare.TNR)\n",
    "        print('for comparison: ' + str(printout))\n",
    "#         for count, metric in enumerate(metricslist):\n",
    "#             D = getattr(proclam.metrics, metric)()\n",
    "#             hm = D.evaluate(probm, truthv)\n",
    "#             data[count][cc] = hm\n",
    "#     dataset['results'] = data\n",
    "    \n",
    "#     metric_plot(dataset, metricslist, markerlist, colors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# more_data = np.empty((len(metricslist), len(more_names)))\n",
    "# for cc, pair in enumerate(more_class_pairs):\n",
    "#     probm, truthv = read_class_pairs(pair, dirname)\n",
    "#     for count, metric in enumerate(metricslist):\n",
    "#         D = getattr(proclam.metrics, metric)()\n",
    "#         hm = D.evaluate(probm, truthv)\n",
    "#         more_data[count][cc] = hm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# metric_plot(more_names, metricslist, more_data, markerlist, colors, title='SNPhotCC', fileloc=dirname+'snphotccdata.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = np.empty((len(metricslist), len(names)))\n",
    "# for cc, pair in enumerate(class_pairs):\n",
    "#     probm, truthv = read_class_pairs(pair, dirname)\n",
    "#     for count, metric in enumerate(metricslist):\n",
    "#         D = getattr(proclam.metrics, metric)()\n",
    "#         hm = D.evaluate(probm, truthv)\n",
    "#         data[count][cc] = hm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# metric_plot(names, metricslist, data, markerlist, colors, title='Mystery Dataset', fileloc=dirname+'mysterydata.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "proclam (Python 3)",
   "language": "python",
   "name": "proclam_3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}