literal_performance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "from glob import glob\n",
    "from collections import Counter, defaultdict, OrderedDict\n",
    "from operator import itemgetter\n",
    "import itertools\n",
    "import numpy as np\n",
    "from scipy.misc import logsumexp\n",
    "from scipy import sparse\n",
    "import matplotlib.pyplot as plot\n",
    "import json\n",
    "import traceback\n",
    "\n",
    "from tuna import *\n",
    "from featurefunctions import cross_product_features, null_features\n",
    "import training_instances as inst\n",
    "from learning import score, cost\n",
    "import config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def evaluate_pretrained(\n",
    "        filenames=glob(\"../TUNA/corpus/singular/furniture/*.xml\"),\n",
    "        phi=cross_product_features,\n",
    "        weights={},\n",
    "        verbose=True,\n",
    "        eval_num='',\n",
    "        options=None):\n",
    "    if options is None:\n",
    "        options = config.options()\n",
    "\n",
    "    get_instances = (inst.get_generation_instances\n",
    "                     if options.generation else\n",
    "                     inst.get_plural_instances\n",
    "                     if 'plural' in options.data_dir else\n",
    "                     inst.get_singular_instances)\n",
    "    D = get_instances(filenames=filenames)\n",
    "    # messages is the set of utterances observed in training, as a proxy for\n",
    "    # the set of all possible utterances. TODO: Can we do this in a more principled way?\n",
    "    messages = [d[0] for d in D]\n",
    "\n",
    "    # Train-test split:\n",
    "    train, test = None, None\n",
    "    if options.train_percentage > 0.0: # where 0, no split:\n",
    "        random.shuffle(D)\n",
    "        train_size = int(round(len(D)*options.train_percentage, 0))\n",
    "        train = D[ : train_size]\n",
    "        test = D[train_size: ]\n",
    "    else:\n",
    "        train = D\n",
    "        test = D\n",
    "\n",
    "    \n",
    "    dump_params(weights, config.get_file_path('params.json'))\n",
    "\n",
    "    # Optionally view weights:\n",
    "    if verbose:\n",
    "        for key, val in sorted(weights.items(), key=itemgetter(1), reverse=True):\n",
    "            if val != 0.0:\n",
    "                print key, val\n",
    "\n",
    "    # Accuracy evaluation:\n",
    "    results = defaultdict(int)\n",
    "    with open(config.get_file_path('predictions.%s.jsons' % eval_num), 'w') as outfile:\n",
    "        for (id, x, y, domain) in test:\n",
    "            prediction = predict(x=x, w=weights, phi=phi,\n",
    "                                 messages=messages,\n",
    "                                 classes=domain)\n",
    "            json.dump({'id': id, 'input': x,\n",
    "                       'gold': y, 'prediction': prediction}, outfile)\n",
    "            outfile.write('\\n')\n",
    "            results[y==prediction] += 1\n",
    "    acc = float(results[True])/len(test)\n",
    "\n",
    "    if verbose:\n",
    "        print \"Accuracy: %s of %s (%0.02f%%)\" % (results[True], len(test), acc)\n",
    "    return acc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Namespace' object has no attribute 'generation'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-d3c076549143>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     15\u001b[0m         \u001b[1;32mprint\u001b[0m \u001b[1;34m'accuracy std: %0.2f'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0maccs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m \u001b[0mmain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-14-d3c076549143>\u001b[0m in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m     10\u001b[0m         accs = np.array([evaluate_pretrained(filenames=filenames, phi=phi, weights=weights,\n\u001b[0;32m     11\u001b[0m                                   eval_num=i, options=options)\n\u001b[1;32m---> 12\u001b[1;33m                          for i in range(1)])\n\u001b[0m\u001b[0;32m     13\u001b[0m         \u001b[1;32mprint\u001b[0m \u001b[1;34m'Finished %d random 80/20 splits using %s on %s'\u001b[0m \u001b[1;33m%\u001b[0m             \u001b[1;33m(\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevaluate_reps\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeatname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata_dir\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m         \u001b[1;32mprint\u001b[0m \u001b[1;34m'accuracy mean: %0.2f'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0maccs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-5-af3ab5f2c82b>\u001b[0m in \u001b[0;36mevaluate_pretrained\u001b[1;34m(filenames, phi, weights, verbose, eval_num, options)\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     get_instances = (inst.get_generation_instances\n\u001b[1;32m---> 12\u001b[1;33m                      \u001b[1;32mif\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgeneration\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m                      \u001b[0minst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_plural_instances\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m                      \u001b[1;32mif\u001b[0m \u001b[1;34m'plural'\u001b[0m \u001b[1;32min\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata_dir\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'Namespace' object has no attribute 'generation'"
     ]
    }
   ],
   "source": [
    "def main():\n",
    "    sys.argv=['program', '--run_dir', './']\n",
    "    options = config.options()\n",
    "    with open('runs/66/params.json', 'r') as infile:\n",
    "        weights = json.load(infile)\n",
    "    filenames = glob(\"../TUNA/corpus/singular/furniture/*.xml\")\n",
    "    for featname, phi in (#('Random', null_features, log_loss_grad),\n",
    "        #('Literal listener', cross_product_features, log_loss_grad),\n",
    "        ('RSA', cross_product_features),):\n",
    "        accs = np.array([evaluate_pretrained(filenames=filenames, phi=phi, weights=weights,\n",
    "                                  eval_num=i, options=options)\n",
    "                         for i in range(1)])\n",
    "        print 'Finished %d random 80/20 splits using %s on %s' % \\\n",
    "            (options.evaluate_reps, featname, options.data_dir)\n",
    "        print 'accuracy mean: %0.2f' % accs.mean()\n",
    "        print 'accuracy std: %0.2f' % accs.std()\n",
    "        \n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}