notebooks/06-Using-Pretrained-Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0, \"..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.tokens import Span\n",
    "\n",
    "import medspacy\n",
    "from medspacy.preprocess import PreprocessingRule, Preprocessor\n",
    "from medspacy.ner import TargetRule\n",
    "from medspacy.context import ConTextRule\n",
    "from medspacy.section_detection import Sectionizer\n",
    "from medspacy.postprocess import PostprocessingRule, PostprocessingPattern, Postprocessor\n",
    "from medspacy.postprocess import postprocessing_functions\n",
    "from medspacy.visualization import visualize_ent, visualize_dep\n",
    "\n",
    "\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Overview\n",
    "In this notebook, we'll show how to use a pretrained model for target concept extraction instead of defining rules. We'll then add our additional components to show how medSpaCy can be used to combine statistical NLP with other rule-based components.\n",
    "\n",
    "As an example, we'll download the [med7](https://github.com/kormilitzin/med7) transformers model which can be used with spacy 3. This won't get all the concepts we're interested in, but will extract drug-related information like names and doses.\n",
    "\n",
    "We can install this model with `pip` using this GitHub link:\n",
    "```bash\n",
    "pip install --no-deps https://huggingface.co/kormilitzin/en_core_med7_trf/resolve/main/en_core_med7_trf-any-py3-none-any.whl\n",
    "```\n",
    "\n",
    "**NOTE:** As of, Oct 2022, med7 supports spacy 3.1 only. Because medspaCy now supports spacy 3.3 or higher, please install the model with no deps to keep the spacy version in the environment the same. The model still functions as of 3.4.1.\n",
    "\n",
    "We'll also need to install `spacy-transformers`:\n",
    "```bash\n",
    "pip install spacy-transformers\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install --no-deps https://huggingface.co/kormilitzin/en_core_med7_trf/resolve/main/en_core_med7_trf-any-py3-none-any.whl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install spacy-transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./discharge_summary.txt\") as f:\n",
    "    text = f.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model now can be loaded as any other spaCy model. We'll use `medspacy.load()` and pass in this model name."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "[E050] Can't find model 'en_core_med7_trf'. It doesn't seem to be a Python package or a valid path to a data directory.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m nlp \u001b[38;5;241m=\u001b[39m \u001b[43mmedspacy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43men_core_med7_trf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Documents/medspacy/notebooks/../medspacy/util.py:77\u001b[0m, in \u001b[0;36mload\u001b[0;34m(model, medspacy_enable, medspacy_disable, load_rules, quickumls_path, **model_kwargs)\u001b[0m\n\u001b[1;32m     75\u001b[0m     nlp \u001b[38;5;241m=\u001b[39m model\n\u001b[1;32m     76\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(model, \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m---> 77\u001b[0m     nlp \u001b[38;5;241m=\u001b[39m \u001b[43mspacy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     78\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     79\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m     80\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel must be either \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdefault\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m or an actual spaCy Language object, not \u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     81\u001b[0m         \u001b[38;5;28mtype\u001b[39m(model),\n\u001b[1;32m     82\u001b[0m     )\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/spacy/__init__.py:54\u001b[0m, in \u001b[0;36mload\u001b[0;34m(name, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload\u001b[39m(\n\u001b[1;32m     31\u001b[0m     name: Union[\u001b[38;5;28mstr\u001b[39m, Path],\n\u001b[1;32m     32\u001b[0m     \u001b[38;5;241m*\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     37\u001b[0m     config: Union[Dict[\u001b[38;5;28mstr\u001b[39m, Any], Config] \u001b[38;5;241m=\u001b[39m util\u001b[38;5;241m.\u001b[39mSimpleFrozenDict(),\n\u001b[1;32m     38\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Language:\n\u001b[1;32m     39\u001b[0m     \u001b[38;5;124;03m\"\"\"Load a spaCy model from an installed package or a local path.\u001b[39;00m\n\u001b[1;32m     40\u001b[0m \n\u001b[1;32m     41\u001b[0m \u001b[38;5;124;03m    name (str): Package name or model path.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[38;5;124;03m    RETURNS (Language): The loaded nlp object.\u001b[39;00m\n\u001b[1;32m     53\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 54\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mutil\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     55\u001b[0m \u001b[43m        \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     56\u001b[0m \u001b[43m        \u001b[49m\u001b[43mvocab\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvocab\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     57\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdisable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdisable\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     58\u001b[0m \u001b[43m        \u001b[49m\u001b[43menable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menable\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     59\u001b[0m \u001b[43m        \u001b[49m\u001b[43mexclude\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexclude\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     60\u001b[0m \u001b[43m        \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     61\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/spacy/util.py:436\u001b[0m, in \u001b[0;36mload_model\u001b[0;34m(name, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[1;32m    434\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m OLD_MODEL_SHORTCUTS:\n\u001b[1;32m    435\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIOError\u001b[39;00m(Errors\u001b[38;5;241m.\u001b[39mE941\u001b[38;5;241m.\u001b[39mformat(name\u001b[38;5;241m=\u001b[39mname, full\u001b[38;5;241m=\u001b[39mOLD_MODEL_SHORTCUTS[name]))  \u001b[38;5;66;03m# type: ignore[index]\u001b[39;00m\n\u001b[0;32m--> 436\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIOError\u001b[39;00m(Errors\u001b[38;5;241m.\u001b[39mE050\u001b[38;5;241m.\u001b[39mformat(name\u001b[38;5;241m=\u001b[39mname))\n",
      "\u001b[0;31mOSError\u001b[0m: [E050] Can't find model 'en_core_med7_trf'. It doesn't seem to be a Python package or a valid path to a data directory."
     ]
    }
   ],
   "source": [
    "nlp = medspacy.load(\"en_core_med7_trf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp.pipe_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ner = nlp.get_pipe(\"ner\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ner.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc.ents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process our text\n",
    "Similar to the last notebook, we'll add new rules to some of our components. Let's first look at what our model extracts out of the box:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visualize_ent(doc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = Preprocessor(nlp.tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp.tokenizer = preprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocess_rules = [\n",
    "    \n",
    "    PreprocessingRule(\n",
    "        r\"\\[\\*\\*[\\d]{1,4}-[\\d]{1,2}(-[\\d]{1,2})?\\*\\*\\]\",\n",
    "        repl=\"01-01-2010\",\n",
    "        desc=\"Replace MIMIC date brackets with a generic date.\"\n",
    "    ),\n",
    "    \n",
    "    PreprocessingRule(\n",
    "        r\"\\[\\*\\*[\\d]{4}\\*\\*\\]\",\n",
    "        repl=\"2010\",\n",
    "        desc=\"Replace MIMIC year brackets with a generic year.\"\n",
    "    ),\n",
    "    \n",
    "    PreprocessingRule(\n",
    "        r\"dx'd\", \n",
    "        repl=\"Diagnosed\", \n",
    "        desc=\"Replace abbreviation\"\n",
    "    ),\n",
    "    \n",
    "    PreprocessingRule(\n",
    "        r\"tx'd\", \n",
    "        repl=\"Treated\", \n",
    "        desc=\"Replace abbreviation\"\n",
    "    ),\n",
    "    \n",
    "    PreprocessingRule(\n",
    "        r\"\\[\\*\\*[^\\]]+\\]\", \n",
    "        desc=\"Remove all other bracketed placeholder text from MIMIC\"\n",
    "    )\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor.add(preprocess_rules)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Target Rules\n",
    "The trained NER will add some new concepts that we weren't getting before, but we can customize with rules."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_rules = [\n",
    "    TargetRule(literal=\"abdominal pain\", category=\"PROBLEM\"),\n",
    "    TargetRule(\"stroke\", \"PROBLEM\"),\n",
    "    TargetRule(\"hemicolectomy\", \"TREATMENT\"),\n",
    "    TargetRule(\"colon cancer\", \"PROBLEM\"),\n",
    "    TargetRule(\"radiotherapy\", \"PROBLEM\",\n",
    "              pattern=[{\"LOWER\": \"xrt\"}]),\n",
    "    TargetRule(\"metastasis\", \"PROBLEM\"),\n",
    "    \n",
    "    TargetRule(\"Type II Diabetes Mellitus\", \"PROBLEM\", \n",
    "              pattern=[\n",
    "                  {\"LOWER\": \"type\"},\n",
    "                  {\"LOWER\": {\"IN\": [\"2\", \"ii\", \"two\"]}},\n",
    "                  {\"LOWER\": {\"IN\": [\"dm\", \"diabetes\"]}},\n",
    "                  {\"LOWER\": \"mellitus\", \"OP\": \"?\"}\n",
    "              ],),\n",
    "    TargetRule(\"Hypertension\", \"PROBLEM\",\n",
    "              pattern=[{\"LOWER\": {\"IN\": [\"htn\", \"hypertension\"]}}],),\n",
    "    \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_matcher = nlp.get_pipe(\"medspacy_target_matcher\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_matcher.add(target_rules)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "context = nlp.get_pipe(\"medspacy_context\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "context_rules = [\n",
    "    ConTextRule(\"diagnosed in <YEAR>\", \"HISTORICAL\", \n",
    "               pattern=[\n",
    "                   {\"LOWER\": \"diagnosed\"},\n",
    "                   {\"LOWER\": \"in\"},\n",
    "                   {\"LOWER\": {\"REGEX\": \"^[\\d]{4}$\"}}\n",
    "               ])\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "context.add(context_rules)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Section detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sectionizer = nlp.add_pipe(\"medspacy_sectionizer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from medspacy.section_detection import SectionRule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "section_rule = SectionRule(\"Brief Hospital Course:\", \"hospital-course\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sectionizer.add(section_rule)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Postprocessing\n",
    "Here, we'll show another example of how postprocessing can be used. The NER component extracts **\"married\"** as a **\"TREATMENT\"** entity. While some might agree with this in a philosophical sense, it doesn't match our clinical definition very well. This shows a challenge of statistical NLP: we have relatively little control over what concepts are extracted by our model. But we can use some postprocessing rules to clean this up.\n",
    "\n",
    "Postprocessing can be used to remove or clean up entities which we know are incorrect. In this example, we'll just remove any entity where the text is **\"married\"**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "postprocessor = nlp.add_pipe(\"medspacy_postprocessor\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "postprocess_rules = [\n",
    "    PostprocessingRule(\n",
    "        patterns=[\n",
    "            PostprocessingPattern(condition=lambda ent: ent.text.lower(), success_value=\"married\"),\n",
    "        ],\n",
    "        action=postprocessing_functions.remove_ent,\n",
    "        description=\"Remove a specific misclassified span of text.\"\n",
    "    ),\n",
    "    \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "postprocessor.add(postprocess_rules)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process our document\n",
    "Now, let's process the text with our complete pipeline and show the results:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp.pipe_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visualize_ent(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "short_text = \"Colon cancer dx'd in [**2554**], tx'd with hemicolectomy, chemo\"\n",
    "short_doc = nlp(short_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visualize_ent(short_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visualize_dep(short_doc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}