99_process_heffernan_ACL_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f64e1fa3",
   "metadata": {},
   "source": [
    "This notebook is just for the records. The output (.csv files) is part of /datasets. \n",
    "\n",
    "It formats the [original data](https://www.cl.cam.ac.uk/~kh562/ProblemSolution/data_upload/) from [Identifying Problems and Solutions in scientific Text](https://link.springer.com/content/pdf/10.1007/s11192-018-2718-6.pdf) 2020 into a dataframe / CSV `/datasets`\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "366ec4f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random, codecs\n",
    "from lxml import etree\n",
    "\n",
    "problem_strings=[l.strip() for l in codecs.open(\"problem_strings.txt\",\"r\",\"utf-8\").readlines()]\n",
    "non_problem_strings=[l.strip() for l in codecs.open(\"non_problem_strings.txt\",\"r\",\"utf-8\").readlines()]\n",
    "problem_heads = [(p.get(\"HEAD\"),p.get(\"HEAD-POS\")) for i,s in enumerate(etree.parse(\"problem_heads.xml\").getroot().findall(\".//SENT\"))for p in s.findall(\".//PROBLEM\") if p.text==problem_strings[i]]\n",
    "non_problem_heads = [(p.get(\"HEAD\"),p.get(\"HEAD-POS\")) for i,s in enumerate(etree.parse(\"non_problem_heads.xml\").getroot().findall(\".//SENT\"))for p in s.findall(\".//NON-PROBLEM\") if p.text==non_problem_strings[i]]\n",
    "problem_lemmas=[\" \".join([l.get(\"lem\") for l in s.findall(\".//lemma\")]) for s in etree.parse(\"problem_strings_rasp.xml\").getroot().findall(\".//sentence\")]\n",
    "non_problem_lemmas=[\" \".join([l.get(\"lem\") for l in s.findall(\".//lemma\")]) for s in etree.parse(\"non_problem_strings_rasp.xml\").getroot().findall(\".//sentence\")]\n",
    "problem_class_labels = [\"problem\"]*500 + [\"non-problem\"]*500\n",
    "\n",
    "solution_strings=[l.strip() for l in codecs.open(\"solution_strings.txt\",\"r\",\"utf-8\").readlines()]\n",
    "non_solution_strings=[l.strip() for l in codecs.open(\"non_solution_strings.txt\",\"r\",\"utf-8\").readlines()]\n",
    "solution_heads = [(p.get(\"HEAD\"),p.get(\"HEAD-POS\")) for i,s in enumerate(etree.parse(\"solution_heads.xml\").getroot().findall(\".//SENT\"))for p in s.findall(\".//SOLUTION\") if p.text==solution_strings[i]]\n",
    "non_solution_heads = [(p.get(\"HEAD\"),p.get(\"HEAD-POS\")) for i,s in enumerate(etree.parse(\"non_solution_heads.xml\").getroot().findall(\".//S\")) for p in s.findall(\".//NON-SOLUTION\") if p.text==non_solution_strings[i]] #+etree.parse(\"non.solutions3.xml\").getroot().findall(\".//A-S\")\n",
    "solution_lemmas=[\" \".join([l.get(\"lem\") for l in s.findall(\".//lemma\")]) for s in etree.parse(\"solution_strings_rasp.xml\").getroot().findall(\".//sentence\")]\n",
    "non_solution_lemmas=[\" \".join([l.get(\"lem\") for l in s.findall(\".//lemma\")]) for s in etree.parse(\"non_solution_strings_rasp.xml\").getroot().findall(\".//sentence\")]\n",
    "solution_class_labels = [\"solution\"]*500 + [\"non-solution\"]*500\n",
    "\n",
    "tmp_transitivity_vector = []\n",
    "sentences = [s.strip() for s in codecs.open(\"problem_strings_candc.txt\",\"r\",\"utf-8\").readlines()]+[s.strip() for s in codecs.open(\"non_problem_strings_candc.txt\",\"r\",\"utf-8\").readlines()]\n",
    "\n",
    "problem_heads[:5], problem_strings[:5], non_problem_heads[:5], non_problem_strings[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eac1bc6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "heads = [s.text for s in etree.parse(\"problem_heads.xml\").getroot().findall(\".//SENT\")]\n",
    "problem_sents = [h + t for h,t in zip(heads, problem_strings) if h and t is not None]\n",
    "len(problem_sents) #500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cad9739",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame({\"text\": problem_sents, \"label\":1}).append(pd.DataFrame({\"text\": non_problem_sents, \"label\":0}), ignore_index=True)\n",
    "df.to_csv (\"problem_nonproblem_ACL_1k.csv\", index = False, header=True)\n",
    "\n",
    "df.sample(frac=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8f1c4eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "np_heads = [s.text for s in etree.parse(\"non_problem_heads.xml\").getroot().findall(\".//SENT\")]\n",
    "non_problem_sents = [h + t for h,t in zip(np_heads, problem_strings) if h and t is not None]\n",
    "\n",
    "solution_heads = [s.text for s in etree.parse(\"solution_heads.xml\").getroot().findall(\".//SENT\")]\n",
    "solution_sents = [h + t for h,t in zip(np_heads, problem_strings) if h and t is not None]\n",
    "\n",
    "non_solution_heads = [s.text for s in etree.parse(\"non_solution_heads.xml\").getroot().findall(\".//SENT\")]\n",
    "non_solution_sents = [h + t for h,t in zip(np_heads, problem_strings) if h and t is not None]\n",
    "\n",
    "df2 = pd.DataFrame({\"text\": solution_sents, \"label\":1}).append(pd.DataFrame({\"text\": non_solution_sents, \"label\":0}), ignore_index=True)\n",
    "df2.to_csv (\"solution_nonsolution_ACL_1k.csv\", index = False, header=True)\n",
    "\n",
    "df2.head()\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}