manual_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "capable-packing",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from src.create_datasets_manual_evaluation import create_verbs_dataset, create_nouns_dataset, create_roles_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "featured-shadow",
   "metadata": {},
   "source": [
    "# 1. Create datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "affecting-organizer",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'workdir/manual_evaluation/data'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "second-mobility",
   "metadata": {},
   "source": [
    "### Verbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adopted-documentary",
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_df, new_df = create_verbs_dataset(results_path='/raid/anwar/generative-ie/workdir/results/test-paper_verbs_st_pattern_vocabfilter',\n",
    "                          predictor=(\"xlnet_embs\",\"gie_swv_test_semiPURExlnet_embs_swvhypers\"),\n",
    "                          gold_dataset_path='workdir/data/swv_gold_dataset.pkl',\n",
    "                          test_indexes_path='workdir/data/swv_gold_dataset_test_split.json',\n",
    "                          frame_description_file='workdir/framenet_data/frame_info.json')\n",
    "\n",
    "\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "obvious-subscription",
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_df.to_csv(f'{data_dir}/verbs-xlnet_embs_k10.csv', index=False)\n",
    "# gold_df.to_pickle(f'{res_dir}/gold_dataset_verbs.pkl')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "noticed-revolution",
   "metadata": {},
   "source": [
    "### NOUNS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "czech-correlation",
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_df, new_df = create_nouns_dataset(results_path='/raid/anwar/generative-ie/workdir/results/test-paper_nouns_st_pattern_stopwords_nounfilter',\n",
    "                          predictor=(\"xlnet_embs\",\"gie_swn_test_semiPURExlnet_embs_swnhypers\"),\n",
    "                          gold_dataset_path='workdir/data/swn_gold_dataset.pkl',\n",
    "                          test_indexes_path='workdir/data/swn_gold_dataset_test_split.json',\n",
    "                          frame_description_file='workdir/framenet_data/frame_info.json'\n",
    "                         )\n",
    "\n",
    "new_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "variable-fields",
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_df.to_csv(f'{data_dir}/nouns-xlnet_embs_k10.csv', index=False)\n",
    "# gold_df.to_pickle(f'{res_dir}/gold_dataset_nouns.pkl')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "uniform-clearance",
   "metadata": {},
   "source": [
    "## Roles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "utility-malpractice",
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_df, new_df = create_roles_dataset(results_path='/raid/anwar/generative-ie/workdir/results/test-paper_roles_st_pattern',\n",
    "                          predictor=(\"xlnet_embs\",\"gie_swr_test_semiPURExlnet_embs_swrhypers\"),\n",
    "                          gold_dataset_path='workdir/data/swr_gold_dataset.pkl',\n",
    "                          test_indexes_path='workdir/data/swr_gold_dataset_test_split.json',\n",
    "                          frame_description_file='workdir/framenet_data/frame_info.json'\n",
    "                         )\n",
    "\n",
    "new_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ancient-constant",
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_df.to_csv(f'{data_dir}/roles-xlnet_embs_k10.csv', index=False)\n",
    "# gold_df.to_pickle(f'{res_dir}/gold_dataset_roles.pkl')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "psychological-religion",
   "metadata": {},
   "source": [
    "# 2. Evaluate manually\n",
    "Manual evaluation can be found here:\n",
    "https://docs.google.com/spreadsheets/d/1me9YNaQpXJZ0p6pupd-IdmXTJ8AxbeavTIndfeROMpA/edit?usp=sharing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sexual-british",
   "metadata": {},
   "source": [
    "# 3. Results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "coral-craft",
   "metadata": {},
   "source": [
    "## 3.1 automatic evaluation with gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "marine-profit",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "# prepare data for final precision calculation \n",
    "column_map ={'does not fit context':'NC', \n",
    "          'fit context NOT frame': 'C', \n",
    "          'fit context AND frame': 'CF', \n",
    "          'match gold': 'G'}\n",
    "\n",
    "def dump_tps_gps_aps(results_path,\n",
    "                     gold_path,\n",
    "                     exp_name,\n",
    "                     save_results_path=None):\n",
    "    \n",
    "    exp_path = f'{results_path}/{exp_name}'\n",
    "    manual_results_path = f'{exp_path}/results.tsv'\n",
    "    if not save_results_path:\n",
    "        save_results_path = exp_path\n",
    "        \n",
    "    gold_df = pd.read_pickle(gold_path)\n",
    "    \n",
    "    temp_df = pd.read_csv(f'{manual_results_path}', sep='\\t', quoting=csv.QUOTE_NONE)\n",
    "    temp_df.fillna(0, inplace=True)\n",
    "    #\n",
    "    temp_df['match gold'] = temp_df['match gold'].apply(lambda x: 0 if x==-1 else x)\n",
    "    # \n",
    "    L = len(temp_df)//10\n",
    "\n",
    "    gold_df = gold_df[:L]\n",
    "\n",
    "    print(len(gold_df), len(temp_df))\n",
    "\n",
    "    k =10\n",
    "\n",
    "\n",
    "    for column, subdir in column_map.items():\n",
    "\n",
    "        print(column, subdir)\n",
    "        save_dir_path = f'{save_results_path}/{subdir}'\n",
    "        if not os.path.exists(save_dir_path):\n",
    "            os.mkdir(save_dir_path)    \n",
    "\n",
    "        tps, aps, gps =[],[],[]\n",
    "        for i in range(0, len(gold_df)):\n",
    "            res = temp_df[column][i*k:i*k+k]\n",
    "            _tps, _gps = [True if r else False for r in res], max(0, len(res))\n",
    "            _aps = []\n",
    "            for j in range(k):\n",
    "                if j < len(res):\n",
    "                    _aps.append(j+1)\n",
    "                else:\n",
    "                    _aps.append(len(res))\n",
    "            aps.append(_aps)\n",
    "            gps.append(_gps)\n",
    "            tps.append(_tps)\n",
    "\n",
    "        tps, gps, aps = tuple(tps), tuple(gps), tuple(aps)    \n",
    "\n",
    "        print('Saving resutls...')\n",
    "        with open(os.path.join(save_dir_path, 'tps.pkl'), 'wb') as f:\n",
    "            pickle.dump(tps, f)\n",
    "\n",
    "        with open(os.path.join(save_dir_path, 'aps.pkl'), 'wb') as f:\n",
    "            pickle.dump(aps, f)\n",
    "\n",
    "        with open(os.path.join(save_dir_path, 'gps.pkl'), 'wb') as f:\n",
    "            pickle.dump(gps, f)    \n",
    "            \n",
    "   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "domestic-accuracy",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !python -m src.run_evaluate --results_path=$save_results_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "binary-protein",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.run_evaluate import load_tps_aps_gps, precision_at_level_hard\n",
    "\n",
    "column_map ={'does not fit context':'NC', \n",
    "          'fit context NOT frame': 'C', \n",
    "          'fit context AND frame': 'CF', \n",
    "          'match gold': 'G'}\n",
    "\n",
    "def evaluate_manual(results_path,\n",
    "                    exp_name,\n",
    "                    save_results_path=None):\n",
    "    \n",
    "    exp_path = f'{results_path}/{exp_name}'\n",
    "    if not save_results_path:\n",
    "        save_results_path = exp_path\n",
    "        \n",
    "    res_df = pd.DataFrame(columns = ['p@1', 'p@3', 'p@5', 'p@10'], index=column_map.keys())\n",
    "\n",
    "    for column, subdir in column_map.items():\n",
    "        print(column, subdir)\n",
    "        save_dir_path = f'{save_results_path}/{subdir}'\n",
    "\n",
    "        annots = load_tps_aps_gps(save_dir_path)\n",
    "\n",
    "        levels = [1,3, 5,10]\n",
    "        k=10\n",
    "    #     exp_name = 'xlm_embs_manual'\n",
    "        annots_df = pd.DataFrame({'tps' : annots[0], 'aps' : annots[1], 'gps' : annots[2]})\n",
    "        annots_df = annots_df[annots_df.gps != 0]\n",
    "        # print(len(annots_df))\n",
    "        annots = annots_df.tps.tolist(), annots_df.aps.tolist(), annots_df.gps.tolist()\n",
    "\n",
    "#         curve = create_precision_recall_curve(annots, exp_name=None, \n",
    "#                                               output_file_path=None)\n",
    "        tps, aps, gps = annots\n",
    "\n",
    "        precs = precision_at_level_hard(tps, levels=levels)\n",
    "        # mean_av_prec = calc_MAP_at_k(tps, gps, k)\n",
    "\n",
    "\n",
    "        metrics = {}\n",
    "        metrics['precisions_at_level'] = {str(lev) : prec for lev, prec in zip(levels, precs)}\n",
    "        # metrics['map'] = mean_av_prec\n",
    "        res = [prec for lev, prec in zip(levels, precs)]\n",
    "        # res.append(mean_av_prec)\n",
    "        print(res)\n",
    "        res_df.loc[column] = res\n",
    "    #     metrics\n",
    "\n",
    "    return res_df\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "recognized-treasury",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'workdir/manual_evaluation/data'\n",
    "res_dir = 'workdir/manual_evaluation/results'\n",
    "\n",
    "dataset, gold_ds  = 'paper_verbs_st', 'gold_dataset_verbs.pkl'\n",
    "# dataset, gold_ds  = 'paper_nouns_st', 'gold_dataset_nouns.pkl'\n",
    "# dataset, gold_ds  = 'paper_roles_st', 'gold_dataset_roles.pkl'\n",
    "\n",
    "exp_name = 'xlnet_embs'\n",
    "results_path = f'{res_dir}/{dataset}'\n",
    "gold_path = f'{data_dir}/{gold_ds}'\n",
    "save_results_path = f'{res_dir}/{dataset}'\n",
    "\n",
    "dump_tps_gps_aps(results_path, gold_path, exp_name, save_results_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "incorrect-upgrade",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate_manual(results_path, exp_name, save_results_path)                   "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}