SoftCosine_Results_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading results\n",
    "Results have the format:\n",
    "\n",
    "    <dict> {conf:table_of_results},\n",
    "\n",
    "where **conf** represents the experimental setting and is the tuple \n",
    "    \n",
    "    (0-remove_stopwords,\n",
    "     1-remove_punctuation,\n",
    "     2-termsim_metric,\n",
    "     3-normalization,\n",
    "     4-termsim_th,\n",
    "     5-synsets_taken,\n",
    "     6-wscheme,\n",
    "     7-sim_metric)\n",
    "     \n",
    "There is another configuration for last experiments with did:\n",
    "Configuration format\n",
    "0. Removing stopwords flag  :  [True, False]\n",
    "\n",
    "1. Removing punctuation flag  :  [True, False]\n",
    "\n",
    "2. Wordnet similarity metrics  :  [\"path\", \"lch\", \"wup\", \"res\", \"jcn\", \"lin\"]\n",
    "\n",
    "3. Features extracted to compute similarity  :  [\"token\", \"lemma\", \"stem\", \"lemmapos\"]\n",
    "\n",
    "4. Features used to extract synsets  :  [\"token\", \"lemma\", \"lemmapos\"]\n",
    "\n",
    "5. Information Content used in some WordNet metrics  :  [\"bnc_ic_2007\", \"bnc_ic_2000\", \"semcor_ic\", \"brown_ic\"]\n",
    "\n",
    "6. Normalization flag : [True, False]\n",
    "\n",
    "7. Term-Term similarity minimum threslhold  :  [0.0, 0.25, 0.5, 0.75]            \n",
    "\n",
    "8. Synsets selection strategy (all-vs-all, first)  :  [\"all\", \"first\"]\n",
    "\n",
    "9. Features weighting scheme  :  [\"tf\", \"binary\"]\n",
    "\n",
    "10. Text similarity method  :  [\"mihalcea\", \"softcosine\", \"stevenson\"]\n",
    "     \n",
    "**table_of_results** is a numpy 2d array with scores for each **similarity treshold** in the range **range(0, 101, 5)** sorted by the best accuracy. The **scores** (columns) are:\n",
    "\n",
    "    scores = (0-threshold,\n",
    "              1-accuracy,\n",
    "              2-f_measure,\n",
    "              3-precision,\n",
    "              4-recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17280"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#experiments = pickle.load(open(\"./Results/results_20170620_complex_normalization.pickle\", 'rb'))\n",
    "experiments = pickle.load(open(\"./Results/results_20170830_normalization_4termsimTh_Rada_Stevenson_Softcosine.pickle\", 'rb'))\n",
    "len(experiments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((False,\n",
       "  True,\n",
       "  'lin',\n",
       "  'lemma',\n",
       "  'lemma',\n",
       "  'bnc_ic_2007',\n",
       "  False,\n",
       "  0.5,\n",
       "  'all',\n",
       "  'binary',\n",
       "  'mihalcea'),\n",
       " 0.7,\n",
       " 0.7409224730127576,\n",
       " 0.8188679245283019)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# res = (threshold, accuracy, f_measure, precision, recall)\n",
    "max([(conf, res[0][0], res[0][1], res[0][2])\n",
    "     for conf, res in experiments.items()\n",
    "     #if conf[2] == \"path\"\n",
    "    ]\n",
    "    , key=lambda x:x[2])\n",
    "#[(conf, res[0][0], res[0][1]) for conf, res in all_scores.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_best_results(experiments, ktop=3, metric=\"accuracy\", wordnet_metric=None, text_metric=None):\n",
    "    \"\"\"Extract best results\"\"\"\n",
    "    # Maps score names to column index in the numpy 2d array\n",
    "    # containing the results for a particular configuration\n",
    "    scores_map = {\"accuracy\":1, \"f_measure\":2, \"precision\":3, \"recall\":4}\n",
    "    \n",
    "    best_results = np.zeros((ktop*2, 5))\n",
    "    corresponding_confs = np.array([(None, None, None, None, None,\n",
    "                                     None, None, None, None, None, None)]*(ktop*2), dtype=np.dtype('O'))\n",
    "    #print(\"best_results:\", best_results.shape)\n",
    "    #print(\"corresponding_confs:\", corresponding_confs.shape)\n",
    "    #print(\"Starts\\n-------------------\")\n",
    "    for conf, results in experiments.items():\n",
    "        # Skip text sim metrics distinct to <text_metric>\n",
    "        if text_metric and conf[10] != text_metric:\n",
    "            continue\n",
    "        # Skip wordnet sim metrics distinct to <wordnet_metric>\n",
    "        if wordnet_metric and conf[2] != wordnet_metric:\n",
    "            continue\n",
    "        \n",
    "        # Sort by desired score in descending order and select best ktop\n",
    "        sorted_results = results[results[:,scores_map[metric]].argsort()[::-1]][:ktop, :]\n",
    "        #print(\"sorted results\", sorted_results.shape)\n",
    "        \n",
    "        # Configuration of results\n",
    "        results_conf = np.array([conf]*ktop, dtype=np.dtype('O'))\n",
    "        #print(\"results conf\", sorted_results.shape)\n",
    "        corresponding_confs[ktop:,:] = results_conf\n",
    "        #print(\"corresponding_confs\", corresponding_confs.shape)\n",
    "        \n",
    "        #print(corresponding_confs)\n",
    "                            \n",
    "        # Vertically stacking best and new results\n",
    "        best_results[ktop:,:] = sorted_results[:ktop,:]\n",
    "        #print(best_results)\n",
    "        \n",
    "        # Updating the best results if needed\n",
    "        idx = best_results[:,scores_map[metric]].argsort(axis=0)[::-1]\n",
    "        \n",
    "        best_results = best_results[idx,:]\n",
    "        corresponding_confs = corresponding_confs[idx,:]\n",
    "        \n",
    "        \n",
    "        #print(\"New best results\")\n",
    "        #print(best_results)\n",
    "        \n",
    "        #print(\"New corresponding configurations\")\n",
    "        #print(corresponding_confs)\n",
    "        \n",
    "        # *** Declare best_results with twice the size of ktop and do everthing in-place ***\n",
    "        # Then compare running times\n",
    "        \n",
    "        # Choose the new corresponding configurations to the new best results\n",
    "        \n",
    "                            \n",
    "        #break\n",
    "        #print(\"\\n\")\n",
    "    \n",
    "    return best_results[:ktop,:], corresponding_confs[:ktop,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.7        0.74092247 0.81886792 0.77575561 0.86705412]\n",
      " [0.7        0.74092247 0.81886792 0.77575561 0.86705412]\n",
      " [0.7        0.74018646 0.81875749 0.77411003 0.86887032]]\n",
      "\n",
      "[[False True 'lin' 'lemma' 'lemma' 'bnc_ic_2007' False 0.5 'all' 'binary'\n",
      "  'mihalcea']\n",
      " [False True 'lin' 'lemma' 'lemma' 'bnc_ic_2007' False 0.5 'all' 'tf'\n",
      "  'mihalcea']\n",
      " [False True 'lin' 'stem' 'token' 'bnc_ic_2007' False 0.5 'all' 'tf'\n",
      "  'mihalcea']]\n"
     ]
    }
   ],
   "source": [
    "bres, bconf = extract_best_results(experiments, ktop=3, metric=\"accuracy\", wordnet_metric=None, text_metric=None)\n",
    "print(bres)\n",
    "print()\n",
    "print(bconf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting some results\n",
    "In this cell we get the best results for each text similarity metric and 3-top best wordnet metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------\n",
      "mihalcea path\n",
      "[[False True 'path' 'stem' 'token' None False 0.5 'all' 'tf' 'mihalcea']]\n",
      "[[0.6        0.7392051  0.82095334 0.76538945 0.88521613]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "mihalcea lch\n",
      "[[False True 'lch' 'token' 'lemma' None True 0.0 'all' 'tf' 'mihalcea']]\n",
      "[[0.7        0.73454367 0.82145215 0.7526459  0.90410461]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "mihalcea wup\n",
      "[[False True 'wup' 'stem' 'token' None False 0.75 'all' 'tf' 'mihalcea']]\n",
      "[[0.65       0.73307164 0.82099375 0.75037594 0.90628405]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "mihalcea res\n",
      "[[False True 'res' 'lemma' 'lemma' 'brown_ic' True 0.0 'all' 'tf'\n",
      "  'mihalcea']]\n",
      "[[0.65       0.73675172 0.8173617  0.7690583  0.87213948]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "mihalcea jcn\n",
      "[[False True 'jcn' 'stem' 'token' 'brown_ic' True 0.5 'all' 'tf'\n",
      "  'mihalcea']]\n",
      "[[0.55       0.73478901 0.82448449 0.74544921 0.92226662]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "mihalcea lin\n",
      "[[False True 'lin' 'lemma' 'lemma' 'bnc_ic_2007' False 0.5 'all' 'tf'\n",
      "  'mihalcea']]\n",
      "[[0.7        0.74092247 0.81886792 0.77575561 0.86705412]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "softcosine path\n",
      "[[False True 'path' 'stem' 'token' None False 0.5 'all' 'binary'\n",
      "  'softcosine']]\n",
      "[[0.6        0.73356232 0.81717172 0.76153122 0.88158373]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "softcosine lch\n",
      "[[False True 'lch' 'lemma' 'lemmapos' None True 0.75 'all' 'binary'\n",
      "  'softcosine']]\n",
      "[[0.55       0.73184495 0.82031892 0.74924925 0.90628405]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "softcosine wup\n",
      "[[False True 'wup' 'stem' 'lemmapos' None False 0.75 'first' 'binary'\n",
      "  'softcosine']]\n",
      "[[0.55       0.72988224 0.81852645 0.74924562 0.90192517]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "softcosine res\n",
      "[[False True 'res' 'stem' 'token' 'bnc_ic_2007' True 0.25 'first'\n",
      "  'binary' 'softcosine']]\n",
      "[[0.55       0.7328263  0.82324298 0.74413146 0.9211769 ]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "softcosine jcn\n",
      "[[False True 'jcn' 'token' 'token' 'brown_ic' True 0.0 'first' 'binary'\n",
      "  'softcosine']]\n",
      "[[0.55       0.73233562 0.81874065 0.75443968 0.89502361]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "softcosine lin\n",
      "[[False True 'lin' 'stem' 'token' 'bnc_ic_2000' False 0.75 'all' 'binary'\n",
      "  'softcosine']]\n",
      "[[0.65       0.73552502 0.813301   0.77722608 0.85288776]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "stevenson path\n",
      "[[False True 'path' 'stem' 'token' None False 0.5 'first' 'binary'\n",
      "  'stevenson']]\n",
      "[[0.55       0.7328263  0.82079974 0.75030084 0.90592081]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "stevenson lch\n",
      "[[False True 'lch' 'stem' 'token' None True 0.75 'first' 'binary'\n",
      "  'stevenson']]\n",
      "[[0.55       0.73159961 0.81911376 0.75174507 0.89974573]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "stevenson wup\n",
      "[[False True 'wup' 'stem' 'lemmapos' None False 0.75 'first' 'binary'\n",
      "  'stevenson']]\n",
      "[[0.55       0.72669284 0.81809275 0.74310294 0.90991645]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "stevenson res\n",
      "[[False True 'res' 'lemma' 'lemma' 'brown_ic' True 0.75 'all' 'binary'\n",
      "  'stevenson']]\n",
      "[[0.55       0.72988224 0.8160401  0.75556931 0.88703233]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "stevenson jcn\n",
      "[[False True 'jcn' 'stem' 'token' 'semcor_ic' True 0.0 'all' 'binary'\n",
      "  'stevenson']]\n",
      "[[0.55       0.73405299 0.82561133 0.74097603 0.9320741 ]]\n",
      "\n",
      "\n",
      "------------------------------------------------------\n",
      "stevenson lin\n",
      "[[False True 'lin' 'token' 'token' 'semcor_ic' False 0.75 'first'\n",
      "  'binary' 'stevenson']]\n",
      "[[0.55       0.73331698 0.82447925 0.74215116 0.92735198]]\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "my_results = []\n",
    "for text_metric in [\"mihalcea\", \"softcosine\", \"stevenson\"]:\n",
    "    for wordnet_metric in [\"path\", \"lch\", \"wup\", \"res\", \"jcn\", \"lin\"]:\n",
    "        print(\"------------------------------------------------------\")\n",
    "        print(text_metric, wordnet_metric)\n",
    "        res, conf = extract_best_results(experiments,\n",
    "                                         ktop=1,\n",
    "                                         metric=\"accuracy\",\n",
    "                                         wordnet_metric=wordnet_metric,\n",
    "                                         text_metric=text_metric)\n",
    "        print(conf)\n",
    "        print(res)\n",
    "        print(\"\\n\")\n",
    "        my_results.append(np.concatenate((conf[0,:], res[0,0:3]), axis=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "res = np.array(sorted(my_results, key=lambda x:x[-2], reverse=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving results to latex table format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "0. Removing stopwords flag  :  [True, False]\n",
    "\n",
    "1. Removing punctuation flag  :  [True, False]\n",
    "\n",
    "2. Wordnet similarity metrics  :  [\"path\", \"lch\", \"wup\", \"res\", \"jcn\", \"lin\"]\n",
    "\n",
    "3. Features extracted to compute similarity  :  [\"token\", \"lemma\", \"stem\", \"lemmapos\"]\n",
    "\n",
    "4. Features used to extract synsets  :  [\"token\", \"lemma\", \"stem\", \"lemmapos\"]\n",
    "\n",
    "5. Information Content used in some WordNet metrics  :  [\"bnc_ic_2007\", \"bnc_ic_2000\", \"semcor_ic\", \"brown_ic\"]\n",
    "\n",
    "6. Normalization flag : [True, False]\n",
    "\n",
    "7. Term-Term similarity minimum threslhold  :  [0.0, 0.25, 0.5, 0.75]            \n",
    "\n",
    "8. Synsets selection strategy (all-vs-all, first)  :  [\"all\", \"first\"]\n",
    "\n",
    "9. Features weighting scheme  :  [\"tf\", \"binary\"]\n",
    "\n",
    "10. Text similarity method  :  [\"mihalcea\", \"softcosine\", \"stevenson\"]\n",
    "\n",
    "11. Text similarity threshold : Real\n",
    "\n",
    "12. Accuracy : Real\n",
    "\n",
    "13. F1-score : Real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ic_map = {None: \"\", \"bnc_ic_2007\": \"bnc07\", \"bnc_ic_2000\": \"bnc00\", \"semcor_ic\":\"semcor\", \"brown_ic\":\"brown\"}\n",
    "ss_map = {\"first\":\"1\", \"all\":\"n\"}\n",
    "ws_map = {\"tf\":\"tf\", \"binary\":\"bin\"}\n",
    "with open(\"latex_out.txt\", \"w\") as fid:\n",
    "    for row in res:\n",
    "        line = \"{0} & {1} & {3} & {9} & {4} & {8} & {2} & {5} & {6} & {7:.2f} & {10} & {11:.2f} & {12:.3f} & {13:.3f}\".format(\n",
    "            r\"\\checkmark\" if row[0] else \"\",    \n",
    "            r\"\\checkmark\" if row[1] else \"\",\n",
    "            row[2],\n",
    "            row[3],\n",
    "            row[4],\n",
    "            ic_map[row[5]],\n",
    "            r\"\\checkmark\" if row[6] else \"\",\n",
    "            row[7],\n",
    "            ss_map[row[8]],\n",
    "            ws_map[row[9]],\n",
    "            row[10],\n",
    "            row[11],\n",
    "            row[12],\n",
    "            row[13]\n",
    "        )+\"\\\\\\\\\\n\"\n",
    "        #line = \" & \".join([str(x) for x in row]) +\"\\\\\\\\\\n\"\n",
    "        fid.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_configurations = []\n",
    "for row in res:\n",
    "    test_configurations.append(tuple(row[:-3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(\"test_configurations.pickle\", \"wb\") as fid:\n",
    "    pickle.dump(test_configurations, fid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyzing test scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_scores = pickle.load(open(\"./Results/test_results_20180504.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(False,\n",
       "  True,\n",
       "  'jcn',\n",
       "  'stem',\n",
       "  'token',\n",
       "  'brown_ic',\n",
       "  True,\n",
       "  0.5,\n",
       "  'all',\n",
       "  'tf',\n",
       "  'mihalcea'): array([[0.55      , 0.74086957, 0.82627283, 0.7454418 , 0.92676548],\n",
       "        [0.6       , 0.73333333, 0.81023102, 0.76898982, 0.85614647],\n",
       "        [0.5       , 0.72463768, 0.82439926, 0.7156611 , 0.97210113],\n",
       "        [0.45      , 0.70956522, 0.81984898, 0.69767442, 0.99389712],\n",
       "        [0.65      , 0.70318841, 0.77081468, 0.79208832, 0.75065388],\n",
       "        [0.4       , 0.68463768, 0.8079096 , 0.67893175, 0.99738448],\n",
       "        [0.35      , 0.67246377, 0.80224011, 0.67017544, 0.99912816],\n",
       "        [0.3       , 0.66608696, 0.79930314, 0.66569936, 1.        ],\n",
       "        [0.25      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.7       , 0.66434783, 0.71631553, 0.81767338, 0.63731473],\n",
       "        [0.75      , 0.6342029 , 0.65910319, 0.86647727, 0.53182214],\n",
       "        [0.8       , 0.56637681, 0.55102041, 0.88439306, 0.40017437],\n",
       "        [0.85      , 0.47710145, 0.36834734, 0.93594306, 0.22929381],\n",
       "        [0.9       , 0.39652174, 0.17446471, 0.96491228, 0.09590235],\n",
       "        [0.95      , 0.34898551, 0.04262575, 0.96153846, 0.02179599],\n",
       "        [1.        , 0.3373913 , 0.00695048, 1.        , 0.00348736]]),\n",
       " (False,\n",
       "  True,\n",
       "  'jcn',\n",
       "  'stem',\n",
       "  'token',\n",
       "  'semcor_ic',\n",
       "  True,\n",
       "  0.0,\n",
       "  'all',\n",
       "  'binary',\n",
       "  'stevenson'): array([[0.55      , 0.73565217, 0.82448037, 0.73811165, 0.93374019],\n",
       "        [0.6       , 0.73217391, 0.81204231, 0.76125095, 0.8700959 ],\n",
       "        [0.5       , 0.72463768, 0.82491707, 0.71455939, 0.97558849],\n",
       "        [0.65      , 0.71362319, 0.78596187, 0.78122308, 0.7907585 ],\n",
       "        [0.45      , 0.70898551, 0.81968391, 0.69700672, 0.99476896],\n",
       "        [0.4       , 0.68521739, 0.80833039, 0.67912218, 0.99825632],\n",
       "        [0.7       , 0.68405797, 0.7432878 , 0.80840164, 0.68788143],\n",
       "        [0.35      , 0.67130435, 0.80167891, 0.66939252, 0.99912816],\n",
       "        [0.3       , 0.66608696, 0.79930314, 0.66569936, 1.        ],\n",
       "        [0.25      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.64811594, 0.68823832, 0.8375    , 0.58413252],\n",
       "        [0.8       , 0.59536232, 0.60430839, 0.86385737, 0.4646905 ],\n",
       "        [0.85      , 0.5315942 , 0.48795944, 0.89327146, 0.33565824],\n",
       "        [0.9       , 0.46144928, 0.33784747, 0.92578125, 0.20662598],\n",
       "        [0.95      , 0.40985507, 0.21329212, 0.93877551, 0.12031386],\n",
       "        [1.        , 0.37507246, 0.11639344, 0.97260274, 0.06190061]]),\n",
       " (False,\n",
       "  True,\n",
       "  'jcn',\n",
       "  'token',\n",
       "  'token',\n",
       "  'brown_ic',\n",
       "  True,\n",
       "  0.0,\n",
       "  'first',\n",
       "  'binary',\n",
       "  'softcosine'): array([[0.55      , 0.73565217, 0.81933439, 0.75090777, 0.90148213],\n",
       "        [0.5       , 0.73043478, 0.82603816, 0.72346003, 0.9625109 ],\n",
       "        [0.6       , 0.72231884, 0.79711986, 0.77512356, 0.82040105],\n",
       "        [0.45      , 0.71014493, 0.81884058, 0.70055797, 0.98517873],\n",
       "        [0.65      , 0.69101449, 0.75449102, 0.79980469, 0.71403662],\n",
       "        [0.4       , 0.68695652, 0.80878187, 0.68097794, 0.9956408 ],\n",
       "        [0.35      , 0.6742029 , 0.80308339, 0.67135325, 0.99912816],\n",
       "        [0.3       , 0.66724638, 0.79986053, 0.66647298, 1.        ],\n",
       "        [0.25      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.7       , 0.65333333, 0.6970618 , 0.83192261, 0.59982563],\n",
       "        [0.75      , 0.61623188, 0.63181313, 0.87250384, 0.49520488],\n",
       "        [0.8       , 0.54086957, 0.50438048, 0.89356984, 0.35135135],\n",
       "        [0.85      , 0.45391304, 0.31640058, 0.94372294, 0.19006103],\n",
       "        [0.9       , 0.37971014, 0.1286645 , 0.97530864, 0.06887533],\n",
       "        [0.95      , 0.34492754, 0.02920962, 1.        , 0.01482127],\n",
       "        [1.        , 0.33681159, 0.00521739, 1.        , 0.00261552]]),\n",
       " (False,\n",
       "  True,\n",
       "  'lch',\n",
       "  'lemma',\n",
       "  'lemmapos',\n",
       "  None,\n",
       "  True,\n",
       "  0.75,\n",
       "  'all',\n",
       "  'binary',\n",
       "  'softcosine'): array([[0.55      , 0.73507246, 0.82099491, 0.74537696, 0.91368788],\n",
       "        [0.6       , 0.73217391, 0.80637049, 0.77643261, 0.83870968],\n",
       "        [0.5       , 0.72521739, 0.82313433, 0.71950424, 0.96163906],\n",
       "        [0.45      , 0.71072464, 0.81966028, 0.7       , 0.98866609],\n",
       "        [0.65      , 0.70318841, 0.76790571, 0.79981114, 0.73844813],\n",
       "        [0.4       , 0.68463768, 0.80736544, 0.67978533, 0.99389712],\n",
       "        [0.7       , 0.6742029 , 0.71984048, 0.84051222, 0.62946818],\n",
       "        [0.35      , 0.67246377, 0.80224011, 0.67017544, 0.99912816],\n",
       "        [0.3       , 0.66666667, 0.79958174, 0.66608595, 1.        ],\n",
       "        [0.25      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.62376812, 0.64321056, 0.87053571, 0.51002616],\n",
       "        [0.8       , 0.54608696, 0.51336234, 0.89393939, 0.36006975],\n",
       "        [0.85      , 0.45855072, 0.32708934, 0.94190871, 0.19790759],\n",
       "        [0.9       , 0.38318841, 0.13776337, 0.97701149, 0.07410636],\n",
       "        [0.95      , 0.34608696, 0.03259005, 1.        , 0.01656495],\n",
       "        [1.        , 0.3373913 , 0.00695048, 1.        , 0.00348736]]),\n",
       " (False,\n",
       "  True,\n",
       "  'lch',\n",
       "  'stem',\n",
       "  'token',\n",
       "  None,\n",
       "  True,\n",
       "  0.75,\n",
       "  'first',\n",
       "  'binary',\n",
       "  'stevenson'): array([[0.55      , 0.73913043, 0.82283465, 0.75017947, 0.91107236],\n",
       "        [0.6       , 0.72695652, 0.80301129, 0.77170418, 0.836966  ],\n",
       "        [0.5       , 0.7257971 , 0.82357329, 0.71968709, 0.9625109 ],\n",
       "        [0.45      , 0.71072464, 0.81952984, 0.70024722, 0.98779425],\n",
       "        [0.65      , 0.69507246, 0.76047359, 0.79599619, 0.72798605],\n",
       "        [0.4       , 0.68695652, 0.8089172 , 0.68076236, 0.99651264],\n",
       "        [0.35      , 0.6742029 , 0.80308339, 0.67135325, 0.99912816],\n",
       "        [0.3       , 0.66724638, 0.79986053, 0.66647298, 1.        ],\n",
       "        [0.25      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.7       , 0.66376812, 0.71028971, 0.83157895, 0.61987794],\n",
       "        [0.75      , 0.62318841, 0.64558342, 0.86171761, 0.51612903],\n",
       "        [0.8       , 0.56231884, 0.54047474, 0.89516129, 0.38709677],\n",
       "        [0.85      , 0.48057971, 0.37864078, 0.92542373, 0.23801221],\n",
       "        [0.9       , 0.4115942 , 0.21378776, 0.95833333, 0.12031386],\n",
       "        [0.95      , 0.36637681, 0.08992506, 1.        , 0.04707934],\n",
       "        [1.        , 0.34666667, 0.03427592, 1.        , 0.01743679]]),\n",
       " (False,\n",
       "  True,\n",
       "  'lch',\n",
       "  'token',\n",
       "  'lemma',\n",
       "  None,\n",
       "  True,\n",
       "  0.0,\n",
       "  'all',\n",
       "  'tf',\n",
       "  'mihalcea'): array([[0.7       , 0.72521739, 0.8125    , 0.74366401, 0.89537925],\n",
       "        [0.65      , 0.71942029, 0.82060786, 0.71373308, 0.96512642],\n",
       "        [0.75      , 0.7084058 , 0.78196792, 0.77758621, 0.7863993 ],\n",
       "        [0.6       , 0.69913043, 0.81444405, 0.69030303, 0.99302528],\n",
       "        [0.55      , 0.68      , 0.80577058, 0.67551622, 0.99825632],\n",
       "        [0.5       , 0.6684058 , 0.80013976, 0.66763848, 0.99825632],\n",
       "        [0.45      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.35      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.3       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.4       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.8       , 0.66028986, 0.70787637, 0.82654249, 0.6190061 ],\n",
       "        [0.85      , 0.57623188, 0.57176333, 0.87142857, 0.42545772],\n",
       "        [0.9       , 0.4626087 , 0.34115139, 0.92307692, 0.2092415 ],\n",
       "        [0.95      , 0.36869565, 0.09776305, 0.98333333, 0.05143854],\n",
       "        [1.        , 0.33681159, 0.00521739, 1.        , 0.00261552]]),\n",
       " (False,\n",
       "  True,\n",
       "  'lin',\n",
       "  'lemma',\n",
       "  'lemma',\n",
       "  'bnc_ic_2007',\n",
       "  False,\n",
       "  0.5,\n",
       "  'all',\n",
       "  'tf',\n",
       "  'mihalcea'): array([[0.65      , 0.73391304, 0.82229965, 0.73955432, 0.92589364],\n",
       "        [0.7       , 0.72405797, 0.80298013, 0.7643814 , 0.84568439],\n",
       "        [0.6       , 0.71536232, 0.8194189 , 0.7086514 , 0.97122929],\n",
       "        [0.55      , 0.70086957, 0.81505376, 0.69202678, 0.9912816 ],\n",
       "        [0.75      , 0.69623188, 0.76225045, 0.79470199, 0.73234525],\n",
       "        [0.5       , 0.68347826, 0.80720339, 0.67833828, 0.99651264],\n",
       "        [0.45      , 0.67188406, 0.80195941, 0.66978375, 0.99912816],\n",
       "        [0.4       , 0.66782609, 0.80013952, 0.66686047, 1.        ],\n",
       "        [0.3       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.35      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.8       , 0.65275362, 0.69391926, 0.8382716 , 0.59197908],\n",
       "        [0.85      , 0.55536232, 0.53711527, 0.87254902, 0.38796861],\n",
       "        [0.9       , 0.46086957, 0.34135977, 0.90943396, 0.21011334],\n",
       "        [0.95      , 0.37217391, 0.10864198, 0.97058824, 0.05754141],\n",
       "        [1.        , 0.3373913 , 0.00695048, 1.        , 0.00348736]]),\n",
       " (False,\n",
       "  True,\n",
       "  'lin',\n",
       "  'stem',\n",
       "  'token',\n",
       "  'bnc_ic_2000',\n",
       "  False,\n",
       "  0.75,\n",
       "  'all',\n",
       "  'binary',\n",
       "  'softcosine'): array([[0.6       , 0.73855072, 0.82265041, 0.74928367, 0.9119442 ],\n",
       "        [0.55      , 0.72985507, 0.82598954, 0.72240366, 0.96425458],\n",
       "        [0.65      , 0.72637681, 0.80251046, 0.77152051, 0.83609416],\n",
       "        [0.5       , 0.71362319, 0.82127352, 0.70191713, 0.98953793],\n",
       "        [0.7       , 0.70898551, 0.77448338, 0.79888786, 0.75152572],\n",
       "        [0.45      , 0.68927536, 0.81033263, 0.68195354, 0.99825632],\n",
       "        [0.75      , 0.67652174, 0.72403561, 0.83657143, 0.63818657],\n",
       "        [0.4       , 0.67478261, 0.80336488, 0.67174678, 0.99912816],\n",
       "        [0.35      , 0.67014493, 0.80125742, 0.66841492, 1.        ],\n",
       "        [0.3       , 0.66608696, 0.79930314, 0.66569936, 1.        ],\n",
       "        [0.25      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.8       , 0.60985507, 0.62381219, 0.86915888, 0.48648649],\n",
       "        [0.85      , 0.51594203, 0.45531637, 0.90414508, 0.30427201],\n",
       "        [0.9       , 0.43130435, 0.25738077, 0.97701149, 0.14821273],\n",
       "        [0.95      , 0.35768116, 0.06576728, 1.        , 0.03400174],\n",
       "        [1.        , 0.33797101, 0.00868056, 1.        , 0.0043592 ]]),\n",
       " (False,\n",
       "  True,\n",
       "  'lin',\n",
       "  'token',\n",
       "  'token',\n",
       "  'semcor_ic',\n",
       "  False,\n",
       "  0.75,\n",
       "  'first',\n",
       "  'binary',\n",
       "  'stevenson'): array([[0.55      , 0.72927536, 0.82003854, 0.73480663, 0.92763731],\n",
       "        [0.6       , 0.72753623, 0.80816327, 0.75978511, 0.86312119],\n",
       "        [0.5       , 0.72231884, 0.82357274, 0.7130102 , 0.97471665],\n",
       "        [0.65      , 0.70898551, 0.78154917, 0.78019114, 0.78291194],\n",
       "        [0.45      , 0.70724638, 0.81801802, 0.69717445, 0.98953793],\n",
       "        [0.7       , 0.68521739, 0.74471086, 0.80816327, 0.69049695],\n",
       "        [0.4       , 0.68231884, 0.80663373, 0.67753408, 0.99651264],\n",
       "        [0.35      , 0.67188406, 0.80195941, 0.66978375, 0.99912816],\n",
       "        [0.3       , 0.66608696, 0.79930314, 0.66569936, 1.        ],\n",
       "        [0.25      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.64695652, 0.68944416, 0.83046683, 0.58936356],\n",
       "        [0.8       , 0.59188406, 0.6036036 , 0.85214626, 0.46730602],\n",
       "        [0.85      , 0.53971014, 0.50436954, 0.88791209, 0.35222319],\n",
       "        [0.9       , 0.47942029, 0.3815427 , 0.90819672, 0.24149956],\n",
       "        [0.95      , 0.41507246, 0.23269962, 0.91071429, 0.13339146],\n",
       "        [1.        , 0.38492754, 0.14504432, 0.95744681, 0.07846556]]),\n",
       " (False,\n",
       "  True,\n",
       "  'path',\n",
       "  'stem',\n",
       "  'token',\n",
       "  None,\n",
       "  False,\n",
       "  0.5,\n",
       "  'all',\n",
       "  'binary',\n",
       "  'softcosine'): array([[0.6       , 0.73797101, 0.8174475 , 0.76147479, 0.88230166],\n",
       "        [0.55      , 0.73333333, 0.82429335, 0.73351462, 0.94071491],\n",
       "        [0.5       , 0.72173913, 0.82404692, 0.71094244, 0.97994769],\n",
       "        [0.65      , 0.71130435, 0.78366638, 0.78095238, 0.7863993 ],\n",
       "        [0.45      , 0.70028986, 0.81529118, 0.69067797, 0.99476896],\n",
       "        [0.7       , 0.68753623, 0.74467077, 0.8153527 , 0.68526591],\n",
       "        [0.4       , 0.68231884, 0.80690627, 0.67711413, 0.99825632],\n",
       "        [0.35      , 0.67304348, 0.80252101, 0.67056758, 0.99912816],\n",
       "        [0.3       , 0.66608696, 0.79930314, 0.66569936, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.64405797, 0.67718191, 0.85298013, 0.56146469],\n",
       "        [0.8       , 0.58492754, 0.57981221, 0.88689408, 0.43068875],\n",
       "        [0.85      , 0.48985507, 0.3956044 , 0.93203883, 0.2510898 ],\n",
       "        [0.9       , 0.4115942 , 0.21011673, 0.97826087, 0.11769834],\n",
       "        [0.95      , 0.34956522, 0.04266212, 1.        , 0.02179599],\n",
       "        [1.        , 0.3373913 , 0.00695048, 1.        , 0.00348736]]),\n",
       " (False,\n",
       "  True,\n",
       "  'path',\n",
       "  'stem',\n",
       "  'token',\n",
       "  None,\n",
       "  False,\n",
       "  0.5,\n",
       "  'all',\n",
       "  'tf',\n",
       "  'mihalcea'): array([[0.6       , 0.7373913 , 0.8175594 , 0.75973054, 0.88491718],\n",
       "        [0.55      , 0.73623188, 0.82626957, 0.73505435, 0.94333043],\n",
       "        [0.5       , 0.72057971, 0.82395909, 0.70898806, 0.98343505],\n",
       "        [0.65      , 0.7142029 , 0.78593139, 0.78287197, 0.78901482],\n",
       "        [0.45      , 0.69971014, 0.815     , 0.69026013, 0.99476896],\n",
       "        [0.7       , 0.68405797, 0.74256023, 0.81030928, 0.68526591],\n",
       "        [0.4       , 0.68289855, 0.80719069, 0.67751479, 0.99825632],\n",
       "        [0.35      , 0.67072464, 0.8013986 , 0.66900175, 0.99912816],\n",
       "        [0.3       , 0.66608696, 0.79930314, 0.66569936, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.64753623, 0.68267223, 0.85045514, 0.57018309],\n",
       "        [0.8       , 0.58144928, 0.57777778, 0.87744227, 0.43068875],\n",
       "        [0.85      , 0.48927536, 0.39781271, 0.92088608, 0.25370532],\n",
       "        [0.9       , 0.4057971 , 0.19733751, 0.96923077, 0.10985179],\n",
       "        [0.95      , 0.35188406, 0.05093379, 0.96774194, 0.02615519],\n",
       "        [1.        , 0.3373913 , 0.00695048, 1.        , 0.00348736]]),\n",
       " (False,\n",
       "  True,\n",
       "  'path',\n",
       "  'stem',\n",
       "  'token',\n",
       "  None,\n",
       "  False,\n",
       "  0.5,\n",
       "  'first',\n",
       "  'binary',\n",
       "  'stevenson'): array([[0.55      , 0.7426087 , 0.82574568, 0.75089222, 0.91717524],\n",
       "        [0.5       , 0.72463768, 0.82335441, 0.71789883, 0.96512642],\n",
       "        [0.6       , 0.72405797, 0.80199667, 0.76690533, 0.84045336],\n",
       "        [0.45      , 0.7084058 , 0.81834597, 0.69852035, 0.98779425],\n",
       "        [0.65      , 0.69913043, 0.76568849, 0.79400749, 0.73931997],\n",
       "        [0.4       , 0.68695652, 0.8089172 , 0.68076236, 0.99651264],\n",
       "        [0.35      , 0.67304348, 0.80252101, 0.67056758, 0.99912816],\n",
       "        [0.7       , 0.66666667, 0.71492315, 0.82873563, 0.62859634],\n",
       "        [0.3       , 0.66666667, 0.79958174, 0.66608595, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.62782609, 0.65446717, 0.85513361, 0.53007847],\n",
       "        [0.8       , 0.56637681, 0.55155875, 0.88291747, 0.40104621],\n",
       "        [0.85      , 0.4857971 , 0.39037801, 0.92207792, 0.24760244],\n",
       "        [0.9       , 0.41971014, 0.23412395, 0.95625   , 0.13339146],\n",
       "        [0.95      , 0.36637681, 0.09143807, 0.98214286, 0.04795118],\n",
       "        [1.        , 0.35304348, 0.05263158, 1.        , 0.02702703]]),\n",
       " (False,\n",
       "  True,\n",
       "  'res',\n",
       "  'lemma',\n",
       "  'lemma',\n",
       "  'brown_ic',\n",
       "  True,\n",
       "  0.0,\n",
       "  'all',\n",
       "  'tf',\n",
       "  'mihalcea'): array([[0.6       , 0.73507246, 0.8260373 , 0.73310811, 0.94594595],\n",
       "        [0.65      , 0.73275362, 0.81298174, 0.76024279, 0.87358326],\n",
       "        [0.55      , 0.71246377, 0.81937363, 0.70356473, 0.98081953],\n",
       "        [0.7       , 0.70318841, 0.77142857, 0.7904849 , 0.7532694 ],\n",
       "        [0.5       , 0.68869565, 0.80937167, 0.68263473, 0.99389712],\n",
       "        [0.45      , 0.67304348, 0.80238262, 0.67076743, 0.99825632],\n",
       "        [0.4       , 0.66782609, 0.80013952, 0.66686047, 1.        ],\n",
       "        [0.35      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.3       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.66202899, 0.7089366 , 0.82943925, 0.6190061 ],\n",
       "        [0.8       , 0.60869565, 0.62353597, 0.86532508, 0.48735833],\n",
       "        [0.85      , 0.5026087 , 0.43478261, 0.88948787, 0.28770706],\n",
       "        [0.9       , 0.41507246, 0.22084942, 0.96621622, 0.12467306],\n",
       "        [0.95      , 0.35130435, 0.04927782, 0.96666667, 0.02528335],\n",
       "        [1.        , 0.3373913 , 0.00695048, 1.        , 0.00348736]]),\n",
       " (False,\n",
       "  True,\n",
       "  'res',\n",
       "  'lemma',\n",
       "  'lemma',\n",
       "  'brown_ic',\n",
       "  True,\n",
       "  0.75,\n",
       "  'all',\n",
       "  'binary',\n",
       "  'stevenson'): array([[0.55      , 0.73391304, 0.81778484, 0.75072886, 0.89799477],\n",
       "        [0.5       , 0.72695652, 0.82326454, 0.7226614 , 0.95640802],\n",
       "        [0.6       , 0.71884058, 0.79317697, 0.77629382, 0.81081081],\n",
       "        [0.45      , 0.71130435, 0.81917211, 0.70192906, 0.98343505],\n",
       "        [0.65      , 0.68637681, 0.74802049, 0.803     , 0.70008718],\n",
       "        [0.4       , 0.68463768, 0.80722892, 0.68      , 0.99302528],\n",
       "        [0.35      , 0.67362319, 0.80266386, 0.67116061, 0.99825632],\n",
       "        [0.3       , 0.66724638, 0.79986053, 0.66647298, 1.        ],\n",
       "        [0.25      , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.7       , 0.65797101, 0.70081136, 0.83757576, 0.60244115],\n",
       "        [0.75      , 0.61855072, 0.63322185, 0.87789799, 0.49520488],\n",
       "        [0.8       , 0.53623188, 0.49748744, 0.88988764, 0.34524847],\n",
       "        [0.85      , 0.44695652, 0.30263158, 0.93665158, 0.18047079],\n",
       "        [0.9       , 0.38550725, 0.14516129, 0.96774194, 0.07846556],\n",
       "        [0.95      , 0.34956522, 0.04266212, 1.        , 0.02179599],\n",
       "        [1.        , 0.33971014, 0.01385281, 1.        , 0.00697472]]),\n",
       " (False,\n",
       "  True,\n",
       "  'res',\n",
       "  'stem',\n",
       "  'token',\n",
       "  'bnc_ic_2007',\n",
       "  True,\n",
       "  0.25,\n",
       "  'first',\n",
       "  'binary',\n",
       "  'softcosine'): array([[0.6       , 0.73681159, 0.81255161, 0.77176471, 0.85789015],\n",
       "        [0.55      , 0.73507246, 0.82362022, 0.73891967, 0.93025283],\n",
       "        [0.5       , 0.72115942, 0.82231252, 0.71346154, 0.97035745],\n",
       "        [0.65      , 0.71188406, 0.77762864, 0.79871324, 0.7576286 ],\n",
       "        [0.45      , 0.70028986, 0.8146289 , 0.69183922, 0.99040976],\n",
       "        [0.4       , 0.68057971, 0.80605421, 0.67591499, 0.99825632],\n",
       "        [0.7       , 0.67130435, 0.72082718, 0.8280543 , 0.63818657],\n",
       "        [0.35      , 0.66956522, 0.80083857, 0.66822157, 0.99912816],\n",
       "        [0.3       , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.62434783, 0.64972973, 0.85490754, 0.52397559],\n",
       "        [0.8       , 0.55594203, 0.52890529, 0.89770355, 0.37489102],\n",
       "        [0.85      , 0.46144928, 0.33309404, 0.94308943, 0.20226678],\n",
       "        [0.9       , 0.38550725, 0.14378029, 0.97802198, 0.07759372],\n",
       "        [0.95      , 0.34434783, 0.02751505, 1.        , 0.01394943],\n",
       "        [1.        , 0.33623188, 0.00348129, 1.        , 0.00174368]]),\n",
       " (False,\n",
       "  True,\n",
       "  'wup',\n",
       "  'stem',\n",
       "  'lemmapos',\n",
       "  None,\n",
       "  False,\n",
       "  0.75,\n",
       "  'first',\n",
       "  'binary',\n",
       "  'softcosine'): array([[0.55      , 0.73565217, 0.82103611, 0.74660956, 0.9119442 ],\n",
       "        [0.5       , 0.72347826, 0.82234637, 0.71781534, 0.9625109 ],\n",
       "        [0.6       , 0.72      , 0.79646018, 0.77079935, 0.8238884 ],\n",
       "        [0.45      , 0.70666667, 0.81719653, 0.69771746, 0.98605057],\n",
       "        [0.65      , 0.69797101, 0.76177412, 0.80096154, 0.72624237],\n",
       "        [0.4       , 0.68115942, 0.80592802, 0.67694132, 0.9956408 ],\n",
       "        [0.35      , 0.66956522, 0.80083857, 0.66822157, 0.99912816],\n",
       "        [0.7       , 0.66724638, 0.71328671, 0.83508772, 0.62249346],\n",
       "        [0.3       , 0.66608696, 0.79930314, 0.66569936, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.6226087 , 0.64171712, 0.87014925, 0.50828248],\n",
       "        [0.8       , 0.54376812, 0.507201  , 0.9       , 0.35309503],\n",
       "        [0.85      , 0.45391304, 0.3173913 , 0.93991416, 0.19093287],\n",
       "        [0.9       , 0.38202899, 0.13333333, 0.98795181, 0.07149085],\n",
       "        [0.95      , 0.34434783, 0.02751505, 1.        , 0.01394943],\n",
       "        [1.        , 0.3373913 , 0.00695048, 1.        , 0.00348736]]),\n",
       " (False,\n",
       "  True,\n",
       "  'wup',\n",
       "  'stem',\n",
       "  'lemmapos',\n",
       "  None,\n",
       "  False,\n",
       "  0.75,\n",
       "  'first',\n",
       "  'binary',\n",
       "  'stevenson'): array([[0.55      , 0.7315942 , 0.82019417, 0.7394958 , 0.9206626 ],\n",
       "        [0.6       , 0.72173913, 0.80066445, 0.76447264, 0.84045336],\n",
       "        [0.5       , 0.72057971, 0.82108389, 0.71493213, 0.96425458],\n",
       "        [0.65      , 0.70376812, 0.77156907, 0.79174312, 0.75239756],\n",
       "        [0.45      , 0.70202899, 0.8148415 , 0.69429098, 0.98605057],\n",
       "        [0.4       , 0.67826087, 0.80450863, 0.6749409 , 0.9956408 ],\n",
       "        [0.35      , 0.67014493, 0.80125742, 0.66841492, 1.        ],\n",
       "        [0.7       , 0.66724638, 0.72189922, 0.81243184, 0.64952049],\n",
       "        [0.3       , 0.66608696, 0.79930314, 0.66569936, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.75      , 0.62376812, 0.65606783, 0.83648649, 0.5396687 ],\n",
       "        [0.8       , 0.57449275, 0.56721698, 0.87613843, 0.41935484],\n",
       "        [0.85      , 0.48753623, 0.40189445, 0.89728097, 0.25893636],\n",
       "        [0.9       , 0.42666667, 0.25583145, 0.93406593, 0.14821273],\n",
       "        [0.95      , 0.38202899, 0.14032258, 0.93548387, 0.07585004],\n",
       "        [1.        , 0.35884058, 0.07370184, 0.93617021, 0.03836094]]),\n",
       " (False,\n",
       "  True,\n",
       "  'wup',\n",
       "  'stem',\n",
       "  'token',\n",
       "  None,\n",
       "  False,\n",
       "  0.75,\n",
       "  'all',\n",
       "  'tf',\n",
       "  'mihalcea'): array([[0.6       , 0.7315942 , 0.8263967 , 0.725     , 0.96076722],\n",
       "        [0.65      , 0.72985507, 0.81493249, 0.74835886, 0.89450741],\n",
       "        [0.7       , 0.71246377, 0.7882152 , 0.77238494, 0.80470793],\n",
       "        [0.55      , 0.70956522, 0.81854401, 0.70012392, 0.98517873],\n",
       "        [0.75      , 0.69507246, 0.75557621, 0.80895522, 0.70880558],\n",
       "        [0.5       , 0.68811594, 0.8086771 , 0.68288288, 0.9912816 ],\n",
       "        [0.45      , 0.67304348, 0.80238262, 0.67076743, 0.99825632],\n",
       "        [0.4       , 0.66956522, 0.80097765, 0.66802563, 1.        ],\n",
       "        [0.35      , 0.66666667, 0.79958174, 0.66608595, 1.        ],\n",
       "        [0.3       , 0.66550725, 0.79902473, 0.66531323, 1.        ],\n",
       "        [0.1       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.15      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.2       , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.25      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.05      , 0.66492754, 0.79874652, 0.66492754, 1.        ],\n",
       "        [0.8       , 0.64173913, 0.67677824, 0.84575163, 0.56408021],\n",
       "        [0.85      , 0.54898551, 0.52328431, 0.88041237, 0.3722755 ],\n",
       "        [0.9       , 0.44811594, 0.30612245, 0.93333333, 0.18308631],\n",
       "        [0.95      , 0.36927536, 0.09933775, 0.98360656, 0.05231037],\n",
       "        [1.        , 0.3373913 , 0.00695048, 1.        , 0.00348736]])}"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ic_map = {None: \"\", \"bnc_ic_2007\": \"bnc07\", \"bnc_ic_2000\": \"bnc00\", \"semcor_ic\":\"semcor\", \"brown_ic\":\"brown\"}\n",
    "ss_map = {\"first\":\"1\", \"all\":\"n\"}\n",
    "ws_map = {\"tf\":\"tf\", \"binary\":\"bin\"}\n",
    "new_scores = []\n",
    "for old_row in res:\n",
    "    conf = tuple(old_row[:-3])\n",
    "    scores_mat = test_scores[conf]\n",
    "    scores = [x for x in scores_mat if x[0] == old_row[-3]][0]\n",
    "    new_scores.append((conf[-1], conf[2], scores[1], scores[2]))\n",
    "new_scores.sort(key=lambda x:x[2], reverse=True)\n",
    "with open(\"latex_test_out.txt\", \"w\") as fid:\n",
    "    for row in new_scores:\n",
    "        line = \"{0} & {1} & {2:.3f} & {3:.3f} \\\\\\\\\\n\".format(*row)\n",
    "        fid.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}