1_12_23.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPoqwZ9KW5cYvE/jjmuNIys",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/FaiqaMehboobAwan/NLP-Projects/blob/main/1_12_23.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import sys\n",
        "import nltk\n",
        "import os\n",
        "import spacy\n",
        "import re\n",
        "import csv\n",
        "!pip install stanfordnlp\n",
        "import stanfordnlp\n",
        "import time\n",
        "!pip install scispacy\n",
        "import scispacy\n",
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "f2OoSoribKi2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "2283f208-7ff2-4858-94a6-2958e979451c"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: stanfordnlp in /usr/local/lib/python3.10/dist-packages (0.2.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from stanfordnlp) (1.23.5)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from stanfordnlp) (3.20.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from stanfordnlp) (2.31.0)\n",
            "Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from stanfordnlp) (2.1.0+cu118)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from stanfordnlp) (4.66.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->stanfordnlp) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->stanfordnlp) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->stanfordnlp) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->stanfordnlp) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->stanfordnlp) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->stanfordnlp) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->stanfordnlp) (2.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->stanfordnlp) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->stanfordnlp) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->stanfordnlp) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->stanfordnlp) (2023.11.17)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.0.0->stanfordnlp) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.0.0->stanfordnlp) (1.3.0)\n",
            "Requirement already satisfied: scispacy in /usr/local/lib/python3.10/dist-packages (0.5.3)\n",
            "Collecting spacy<3.7.0,>=3.6.0 (from scispacy)\n",
            "  Downloading spacy-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.6/6.6 MB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy<1.11 in /usr/local/lib/python3.10/dist-packages (from scispacy) (1.10.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scispacy) (2.31.0)\n",
            "Requirement already satisfied: conllu in /usr/local/lib/python3.10/dist-packages (from scispacy) (4.5.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from scispacy) (1.23.5)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from scispacy) (1.3.2)\n",
            "Requirement already satisfied: nmslib>=1.7.3.6 in /usr/local/lib/python3.10/dist-packages (from scispacy) (2.1.1)\n",
            "Requirement already satisfied: scikit-learn>=0.20.3 in /usr/local/lib/python3.10/dist-packages (from scispacy) (1.2.2)\n",
            "Requirement already satisfied: pysbd in /usr/local/lib/python3.10/dist-packages (from scispacy) (0.3.4)\n",
            "Requirement already satisfied: pybind11<2.6.2 in /usr/local/lib/python3.10/dist-packages (from nmslib>=1.7.3.6->scispacy) (2.6.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from nmslib>=1.7.3.6->scispacy) (5.9.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.0.0->scispacy) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.0.0->scispacy) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.0.0->scispacy) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.0.0->scispacy) (2023.11.17)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.3->scispacy) (3.2.0)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (3.0.9)\n",
            "Collecting thinc<8.2.0,>=8.1.8 (from spacy<3.7.0,>=3.6.0->scispacy)\n",
            "  Downloading thinc-8.1.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (919 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m919.6/919.6 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (0.10.1)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (2.0.10)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (0.3.2)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (0.10.3)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (4.66.1)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (1.8.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (3.1.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (23.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->scispacy) (3.3.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->scispacy) (4.5.0)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.7.0,>=3.6.0->scispacy) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.7.0,>=3.6.0->scispacy) (0.1.4)\n",
            "Requirement already satisfied: click<7.2.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy<3.7.0,>=3.6.0->scispacy) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<3.7.0,>=3.6.0->scispacy) (2.1.3)\n",
            "Installing collected packages: thinc, spacy\n",
            "  Attempting uninstall: thinc\n",
            "    Found existing installation: thinc 8.0.17\n",
            "    Uninstalling thinc-8.0.17:\n",
            "      Successfully uninstalled thinc-8.0.17\n",
            "  Attempting uninstall: spacy\n",
            "    Found existing installation: spacy 3.0.9\n",
            "    Uninstalling spacy-3.0.9:\n",
            "      Successfully uninstalled spacy-3.0.9\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "en-core-sci-md 0.4.0 requires spacy<3.1.0,>=3.0.1, but you have spacy 3.6.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed spacy-3.6.1 thinc-8.1.12\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "spacy",
                  "thinc"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6qDNuCmkbBbL",
        "outputId": "d0be57b5-f973-49fa-fbf0-ed6eac0902c9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "dataDS=pd.read_csv('/content/drive/MyDrive/Data/75DS.csv')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataDS.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "uBcuqHV8baA6",
        "outputId": "863a484a-7303-4152-fcc8-19dcee6ae9d7"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  SUBJECT_ID                                               text  \\\n",
              "0      52247  Admission Date:  [**2198-7-4**]              D...   \n",
              "1      57181  Admission Date:  [**2171-10-24**]             ...   \n",
              "2      26896  Admission Date:  [**2129-3-22**]              ...   \n",
              "3      30139  Admission Date:  [**2133-1-19**]              ...   \n",
              "4      29504  Admission Date:  [**2130-1-17**]              ...   \n",
              "\n",
              "            CATEGORY Unnamed: 3 Unnamed: 4  \n",
              "0  Discharge summary        NaN        NaN  \n",
              "1  Discharge summary        NaN        NaN  \n",
              "2  Discharge summary        NaN        NaN  \n",
              "3  Discharge summary        NaN        NaN  \n",
              "4  Discharge summary        NaN        NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bdc73d88-5daa-4031-8cd6-f18b19b3212a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>SUBJECT_ID</th>\n",
              "      <th>text</th>\n",
              "      <th>CATEGORY</th>\n",
              "      <th>Unnamed: 3</th>\n",
              "      <th>Unnamed: 4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>52247</td>\n",
              "      <td>Admission Date:  [**2198-7-4**]              D...</td>\n",
              "      <td>Discharge summary</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>57181</td>\n",
              "      <td>Admission Date:  [**2171-10-24**]             ...</td>\n",
              "      <td>Discharge summary</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>26896</td>\n",
              "      <td>Admission Date:  [**2129-3-22**]              ...</td>\n",
              "      <td>Discharge summary</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>30139</td>\n",
              "      <td>Admission Date:  [**2133-1-19**]              ...</td>\n",
              "      <td>Discharge summary</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>29504</td>\n",
              "      <td>Admission Date:  [**2130-1-17**]              ...</td>\n",
              "      <td>Discharge summary</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bdc73d88-5daa-4031-8cd6-f18b19b3212a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bdc73d88-5daa-4031-8cd6-f18b19b3212a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bdc73d88-5daa-4031-8cd6-f18b19b3212a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-daadeb7d-ddf9-4914-897f-761abbefe13c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-daadeb7d-ddf9-4914-897f-761abbefe13c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-daadeb7d-ddf9-4914-897f-761abbefe13c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Keep only the selected columns\n",
        "dataDS= dataDS[['SUBJECT_ID','text','CATEGORY']]\n",
        "# Print the filtered DataFrame\n",
        "dataDS .head(1)\n",
        "dataDS.to_csv(\"/content/drive/MyDrive/Data/data75DS\", index=False)"
      ],
      "metadata": {
        "id": "3_qZExBNVDfh"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "'''\n",
        "# Compile the regular expression pattern\n",
        "pattern = re.compile(r'(?:^|\\.)\\s*([^:]+)\\s*:\\s*([^.:]+)(?=(?:\\.|$))')\n",
        "\n",
        "# Assuming dataDS and 'TEXT' column are defined\n",
        "docs = dataDS.loc[:, \"TEXT\"].squeeze()\n",
        "\n",
        "# Create a list to store the results\n",
        "result_list = []\n",
        "\n",
        "for i, doc in enumerate(docs):\n",
        "    print(\"Processing \", i, \"th document now\\n\")\n",
        "\n",
        "    # Find all matches in the document\n",
        "    matches = pattern.findall(doc)\n",
        "\n",
        "    # Process each match and append to the result list\n",
        "    for match in matches:\n",
        "        key, value = match\n",
        "        result_list.append({\"Key\": key, \"Value\": value})\n",
        "\n",
        "# Specify the CSV file path\n",
        "csv_file_path = \"output.csv\"\n",
        "\n",
        "# Write the results to a CSV file\n",
        "with open(csv_file_path, 'w', newline='') as csv_file:\n",
        "    fieldnames = [\"Key\", \"Value\"]\n",
        "    writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n",
        "\n",
        "    # Write the header\n",
        "    writer.writeheader()\n",
        "\n",
        "    # Write the data\n",
        "    for result in result_list:\n",
        "        writer.writerow(result)\n",
        "\n",
        "print(f\"Results saved to {csv_file_path}\")\n",
        "'''\n",
        "\n",
        "##useless code but still there to sprinkle salt\n",
        "'''\n",
        "import re\n",
        "pattern = re.compile(r'(?:^|\\.)\\s*([^:]+)\\s*:\\s*([^.:]+)(?=(?:\\.|$))')\n",
        "docs = dataDS.loc[:,\"TEXT\"].squeeze()\n",
        "for i, doc in enumerate(docs):\n",
        "  print(\"Processing \",i,\"th document now\\n\")\n",
        "  matches = pattern.findall(doc)\n",
        "  for match in matches:\n",
        "    key, value = match\n",
        "    print(f\"Key: {key}, Value: {value}\")\n",
        "'''"
      ],
      "metadata": {
        "id": "HtPGhZexioPC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.4.0/en_core_sci_md-0.4.0.tar.gz"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "yNZ-wTTFBNjF",
        "outputId": "ab867b6a-456d-4cf1-acd0-d9a1b1283b56"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.4.0/en_core_sci_md-0.4.0.tar.gz\n",
            "  Downloading https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.4.0/en_core_sci_md-0.4.0.tar.gz (125.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m125.2/125.2 MB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting spacy<3.1.0,>=3.0.1 (from en-core-sci-md==0.4.0)\n",
            "  Downloading spacy-3.0.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy-legacy<3.1.0,>=3.0.5 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (3.0.12)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (3.0.9)\n",
            "Collecting thinc<8.1.0,>=8.0.3 (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0)\n",
            "  Downloading thinc-8.0.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (659 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m659.5/659.5 kB\u001b[0m \u001b[31m40.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (0.7.11)\n",
            "Collecting wasabi<1.1.0,>=0.8.1 (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0)\n",
            "  Downloading wasabi-0.10.1-py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.4 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (2.0.10)\n",
            "Collecting typer<0.4.0,>=0.3.0 (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0)\n",
            "  Downloading typer-0.3.2-py3-none-any.whl (21 kB)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (0.10.3)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (4.66.1)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (1.23.5)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (2.31.0)\n",
            "Collecting pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0)\n",
            "  Downloading pydantic-1.8.2-py3-none-any.whl (126 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.0/126.0 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (3.1.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (23.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (2023.11.17)\n",
            "Collecting click<7.2.0,>=7.1.1 (from typer<0.4.0,>=0.3.0->spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0)\n",
            "  Downloading click-7.1.2-py2.py3-none-any.whl (82 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.8/82.8 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<3.1.0,>=3.0.1->en-core-sci-md==0.4.0) (2.1.3)\n",
            "Building wheels for collected packages: en-core-sci-md\n",
            "  Building wheel for en-core-sci-md (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for en-core-sci-md: filename=en_core_sci_md-0.4.0-py3-none-any.whl size=125733909 sha256=0cff0f99dc937171e2ca63b0143005f712b2f09a72f1d4b24b025754971d5ce6\n",
            "  Stored in directory: /root/.cache/pip/wheels/a6/d7/71/c25351330ce5efecc088e26cffb4dfd90e1a1713a514ffa1db\n",
            "Successfully built en-core-sci-md\n",
            "Installing collected packages: wasabi, pydantic, click, typer, thinc, spacy, en-core-sci-md\n",
            "  Attempting uninstall: wasabi\n",
            "    Found existing installation: wasabi 1.1.2\n",
            "    Uninstalling wasabi-1.1.2:\n",
            "      Successfully uninstalled wasabi-1.1.2\n",
            "  Attempting uninstall: pydantic\n",
            "    Found existing installation: pydantic 1.10.13\n",
            "    Uninstalling pydantic-1.10.13:\n",
            "      Successfully uninstalled pydantic-1.10.13\n",
            "  Attempting uninstall: click\n",
            "    Found existing installation: click 8.1.7\n",
            "    Uninstalling click-8.1.7:\n",
            "      Successfully uninstalled click-8.1.7\n",
            "  Attempting uninstall: typer\n",
            "    Found existing installation: typer 0.9.0\n",
            "    Uninstalling typer-0.9.0:\n",
            "      Successfully uninstalled typer-0.9.0\n",
            "  Attempting uninstall: thinc\n",
            "    Found existing installation: thinc 8.1.12\n",
            "    Uninstalling thinc-8.1.12:\n",
            "      Successfully uninstalled thinc-8.1.12\n",
            "  Attempting uninstall: spacy\n",
            "    Found existing installation: spacy 3.6.1\n",
            "    Uninstalling spacy-3.6.1:\n",
            "      Successfully uninstalled spacy-3.6.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "lida 0.0.10 requires fastapi, which is not installed.\n",
            "lida 0.0.10 requires kaleido, which is not installed.\n",
            "lida 0.0.10 requires python-multipart, which is not installed.\n",
            "lida 0.0.10 requires uvicorn, which is not installed.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "llmx 0.0.15a0 requires openai, which is not installed.\n",
            "llmx 0.0.15a0 requires tiktoken, which is not installed.\n",
            "flask 2.2.5 requires click>=8.0, but you have click 7.1.2 which is incompatible.\n",
            "dask 2023.8.1 requires click>=8.0, but you have click 7.1.2 which is incompatible.\n",
            "distributed 2023.8.1 requires click>=8.0, but you have click 7.1.2 which is incompatible.\n",
            "en-core-web-sm 3.6.0 requires spacy<3.7.0,>=3.6.0, but you have spacy 3.0.9 which is incompatible.\n",
            "fiona 1.9.5 requires click~=8.0, but you have click 7.1.2 which is incompatible.\n",
            "inflect 7.0.0 requires pydantic>=1.9.1, but you have pydantic 1.8.2 which is incompatible.\n",
            "pip-tools 6.13.0 requires click>=8, but you have click 7.1.2 which is incompatible.\n",
            "scispacy 0.5.3 requires spacy<3.7.0,>=3.6.0, but you have spacy 3.0.9 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed click-7.1.2 en-core-sci-md-0.4.0 pydantic-1.8.2 spacy-3.0.9 thinc-8.0.17 typer-0.3.2 wasabi-0.10.1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "click",
                  "pydantic",
                  "spacy",
                  "thinc",
                  "wasabi"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#heuristic_tokenizer code starts here\n",
        "def strip(s):\n",
        "    return s.strip()\n",
        "\n",
        "def is_inline_title(text):\n",
        "    m = re.search('^([a-zA-Z ]+:) ', text)\n",
        "    if not m:\n",
        "        return False\n",
        "    return is_title(m.groups()[0])\n",
        "\n",
        "stopwords = set(['of', 'on', 'or'])\n",
        "def is_title(text):\n",
        "    if not text.endswith(':'):\n",
        "        return False\n",
        "    text = text[:-1]\n",
        "\n",
        "    # be a little loose here... can tighten if it causes errors\n",
        "    text = re.sub('(\\([^\\)]*?\\))', '', text)\n",
        "\n",
        "    # Are all non-stopwords capitalized?\n",
        "    for word in text.split():\n",
        "        if word in stopwords: continue\n",
        "        if not word[0].isupper():\n",
        "            return False\n",
        "\n",
        "    # I noticed this is a common issue (non-title aapears at beginning of line)\n",
        "    if text == 'Disp':\n",
        "        return False\n",
        "\n",
        "    # optionally: could assert that it is less than 6 tokens\n",
        "    return True\n",
        "\n",
        "\n",
        "\n",
        "def sent_tokenize_rules(text):\n",
        "\n",
        "    # long sections are OBVIOUSLY different sentences\n",
        "    text = re.sub('---+', '\\n\\n-----\\n\\n', text)\n",
        "    text = re.sub('___+', '\\n\\n_____\\n\\n', text)\n",
        "    text = re.sub('\\n\\n+', '\\n\\n', text)\n",
        "\n",
        "    segments = text.split('\\n\\n')\n",
        "\n",
        "    # strategy: break down segments and chip away structure until just prose.\n",
        "    #           once you have prose, use nltk.sent_tokenize()\n",
        "\n",
        "    ### Separate section headers ###\n",
        "    new_segments = []\n",
        "\n",
        "    # deal with this one edge case (multiple headers per line) up front\n",
        "    m1 = re.match('(Admission Date:) (.*) (Discharge Date:) (.*)', segments[0])\n",
        "    if m1:\n",
        "        new_segments += list(map(strip,m1.groups()))\n",
        "        segments = segments[1:]\n",
        "\n",
        "    m2 = re.match('(Date of Birth:) (.*) (Sex:) (.*)'            , segments[0])\n",
        "    if m2:\n",
        "        new_segments += list(map(strip,m2.groups()))\n",
        "        segments = segments[1:]\n",
        "\n",
        "    for segment in segments:\n",
        "        # find all section headers\n",
        "        possible_headers  = re.findall('\\n([A-Z][^\\n:]+:)', '\\n'+segment)\n",
        "        #assert len(possible_headers) < 2, str(possible_headers)\n",
        "        headers = []\n",
        "        for h in possible_headers:\n",
        "            #print 'cand=[%s]' % h\n",
        "            if is_title(h.strip()):\n",
        "                #print '\\tYES=[%s]' % h\n",
        "                headers.append(h.strip())\n",
        "\n",
        "        # split text into new segments, delimiting on these headers\n",
        "        for h in headers:\n",
        "            h = h.strip()\n",
        "\n",
        "            # split this segment into 3 smaller segments\n",
        "            ind = segment.index(h)\n",
        "            prefix = segment[:ind].strip()\n",
        "            rest   = segment[ ind+len(h):].strip()\n",
        "\n",
        "            # add the prefix (potentially empty)\n",
        "            if len(prefix) > 0:\n",
        "                new_segments.append(prefix.strip())\n",
        "\n",
        "            # add the header\n",
        "            new_segments.append(h)\n",
        "\n",
        "            # remove the prefix from processing (very unlikely to be empty)\n",
        "            segment = rest.strip()\n",
        "\n",
        "        # add the final piece (aka what comes after all headers are processed)\n",
        "        if len(segment) > 0:\n",
        "            new_segments.append(segment.strip())\n",
        "\n",
        "    # copy over the new list of segments (further segmented than original segments)\n",
        "    segments = list(new_segments)\n",
        "    new_segments = []\n",
        "\n",
        "\n",
        "    ### Low-hanging fruit: \"_____\" is a delimiter\n",
        "    for segment in segments:\n",
        "        subsections = segment.split('\\n_____\\n')\n",
        "        new_segments.append(subsections[0])\n",
        "        for ss in subsections[1:]:\n",
        "            new_segments.append('_____')\n",
        "            new_segments.append(ss)\n",
        "\n",
        "    segments = list(new_segments)\n",
        "    new_segments = []\n",
        "\n",
        "\n",
        "    ### Low-hanging fruit: \"-----\" is a delimiter\n",
        "    for segment in segments:\n",
        "        subsections = segment.split('\\n-----\\n')\n",
        "        new_segments.append(subsections[0])\n",
        "        for ss in subsections[1:]:\n",
        "            new_segments.append('-----')\n",
        "            new_segments.append(ss)\n",
        "\n",
        "    segments = list(new_segments)\n",
        "    new_segments = []\n",
        "\n",
        "    '''\n",
        "    for segment in segments:\n",
        "        print '------------START------------'\n",
        "        print segment\n",
        "        print '-------------END-------------'\n",
        "        print\n",
        "    exit()\n",
        "    '''\n",
        "\n",
        "    ### Separate enumerated lists ###\n",
        "    for segment in segments:\n",
        "        if not re.search('\\n\\s*\\d+\\.', '\\n'+segment):\n",
        "            new_segments.append(segment)\n",
        "            continue\n",
        "\n",
        "        '''\n",
        "        print '------------START------------'\n",
        "        print segment\n",
        "        print '-------------END-------------'\n",
        "        print\n",
        "        '''\n",
        "\n",
        "        # generalizes in case the list STARTS this section\n",
        "        segment = '\\n'+segment\n",
        "\n",
        "        # determine whether this segment contains a bulleted list (assumes i,i+1,...,n)\n",
        "        start = int(re.search('\\n\\s*(\\d+)\\.', '\\n'+segment).groups()[0])\n",
        "        n = start\n",
        "        while re.search('\\n\\s*%d.'%n,segment): # SHOULD CHANGE TO: while re.search('\\n\\s*%d\\.'%n,segment): #(CHANGED . to \\.)\n",
        "            n += 1\n",
        "        n -= 1\n",
        "\n",
        "        # no bulleted list\n",
        "        if n < 1:\n",
        "            new_segments.append(segment)\n",
        "            continue\n",
        "\n",
        "        '''\n",
        "        print '------------START------------'\n",
        "        print segment\n",
        "        print '-------------END-------------'\n",
        "\n",
        "        print start,n\n",
        "        print\n",
        "        '''\n",
        "\n",
        "        # break each list into its own line\n",
        "        # challenge: not clear how to tell when the list ends if more text happens next\n",
        "        for i in range(start,n+1):\n",
        "            matching_text = re.search('(\\n\\s*\\d+\\.)',segment).groups()[0]\n",
        "            prefix  = segment[:segment.index(matching_text) ].strip()\n",
        "            segment = segment[ segment.index(matching_text):].strip()\n",
        "            if len(prefix)>0:\n",
        "                new_segments.append(prefix)\n",
        "\n",
        "        if len(segment)>0:\n",
        "            new_segments.append(segment)\n",
        "\n",
        "    segments = list(new_segments)\n",
        "    new_segments = []\n",
        "\n",
        "    '''\n",
        "        TODO: Big Challenge\n",
        "\n",
        "        There is so much variation in what makes a list. Intuitively, I can tell it's a\n",
        "        list because it shows repeated structure (often following a header)\n",
        "\n",
        "        Examples of some lists (with numbers & symptoms changed around to noise)\n",
        "\n",
        "            Past Medical History:\n",
        "            -- Hyperlipidemia\n",
        "            -- lactose intolerance\n",
        "            -- Hypertension\n",
        "\n",
        "\n",
        "            Physical Exam:\n",
        "            Vitals - T 82.2 BP 123/23 HR 73 R 21 75% on 2L NC\n",
        "            General - well appearing male, sitting up in chair in NAD\n",
        "            Neck - supple, JVP elevated to angle of jaw\n",
        "            CV - distant heart sounds, RRR, faint __PHI_43__ murmur at\n",
        "\n",
        "\n",
        "            Labs:\n",
        "            __PHI_10__ 12:00PM BLOOD WBC-8.8 RBC-8.88* Hgb-88.8* Hct-88.8*\n",
        "            MCV-88 MCH-88.8 MCHC-88.8 RDW-88.8* Plt Ct-888\n",
        "            __PHI_14__ 04:54AM BLOOD WBC-8.8 RBC-8.88* Hgb-88.8* Hct-88.8*\n",
        "            MCV-88 MCH-88.8 MCHC-88.8 RDW-88.8* Plt Ct-888\n",
        "            __PHI_23__ 03:33AM BLOOD WBC-8.8 RBC-8.88* Hgb-88.8* Hct-88.8*\n",
        "            MCV-88 MCH-88.8 MCHC-88.8 RDW-88.8* Plt Ct-888\n",
        "            __PHI_109__ 03:06AM BLOOD WBC-8.8 RBC-8.88* Hgb-88.8* Hct-88.8*\n",
        "            MCV-88 MCH-88.8 MCHC-88.8 RDW-88.8* Plt Ct-888\n",
        "            __PHI_1__ 05:09AM BLOOD WBC-8.8 RBC-8.88* Hgb-88.8* Hct-88.8*\n",
        "            MCV-88 MCH-88.8 MCHC-88.8 RDW-88.8* Plt Ct-888\n",
        "            __PHI_26__ 04:53AM BLOOD WBC-8.8 RBC-8.88* Hgb-88.8* Hct-88.8*\n",
        "            MCV-88 MCH-88.8 MCHC-88.8 RDW-88.8* Plt Ct-888\n",
        "            __PHI_301__ 05:30AM BLOOD WBC-8.8 RBC-8.88* Hgb-88.8* Hct-88.8*\n",
        "            MCV-88 MCH-88.8 MCHC-88.8 RDW-88.8* Plt Ct-888\n",
        "\n",
        "\n",
        "            Medications on Admission:\n",
        "            Allopurinol 100 mg DAILY\n",
        "            Aspirin 250 mg DAILY\n",
        "            Atorvastatin 10 mg DAILY\n",
        "            Glimepiride 1 mg once a week.\n",
        "            Hexavitamin DAILY\n",
        "            Lasix 50mg M-W-F; 60mg T-Th-Sat-Sun\n",
        "            Metoprolol 12.5mg TID\n",
        "            Prilosec OTC 20 mg once a day\n",
        "            Verapamil 120 mg SR DAILY\n",
        "    '''\n",
        "\n",
        "    ### Remove lines with inline titles from larger segments (clearly nonprose)\n",
        "    for segment in segments:\n",
        "        '''\n",
        "        With: __PHI_6__, MD __PHI_5__\n",
        "        Building: De __PHI_45__ Building (__PHI_32__ Complex) __PHI_87__\n",
        "        Campus: WEST\n",
        "        '''\n",
        "\n",
        "        lines = segment.split('\\n')\n",
        "\n",
        "        buf = []\n",
        "        for i in range(len(lines)):\n",
        "            if is_inline_title(lines[i]):\n",
        "                if len(buf) > 0:\n",
        "                    new_segments.append('\\n'.join(buf))\n",
        "                buf = []\n",
        "            buf.append(lines[i])\n",
        "        if len(buf) > 0:\n",
        "            new_segments.append('\\n'.join(buf))\n",
        "\n",
        "    segments = list(new_segments)\n",
        "    new_segments = []\n",
        "\n",
        "\n",
        "    # Going to put one-liner answers with their sections\n",
        "    # (aka A A' B B' C D D' -->  AA' BB' C DD' )\n",
        "    N = len(segments)\n",
        "    for i in range(len(segments)):\n",
        "        # avoid segfaults\n",
        "        if i==0:\n",
        "            new_segments.append(segments[i])\n",
        "            continue\n",
        "\n",
        "        if  segments[i].count('\\n') == 0  and \\\n",
        "               is_title(segments[i-1]) and \\\n",
        "           not is_title(segments[i  ]):\n",
        "            if (i == N-1) or is_title(segments[i+1]):\n",
        "                new_segments = new_segments[:-1]\n",
        "                new_segments.append(segments[i-1] + ' ' + segments[i])\n",
        "            #else: new_segments.append(segments[i]) #ADD TO FIX BUG\n",
        "            # currently If the code sees a segment that doesn't have any new lines and the prior line is a title\n",
        "            # *but* it is not the last segment and the next segment is not a title then that segment is just dropped\n",
        "            # so lists that have a title header will lose their first entry\n",
        "        else:\n",
        "            new_segments.append(segments[i])\n",
        "\n",
        "    segments = list(new_segments)\n",
        "    new_segments = []\n",
        "\n",
        "    '''\n",
        "        Should do some kind of regex to find \"TEST: value\" in segments?\n",
        "\n",
        "            Indication: Source of embolism.\n",
        "            BP (mm Hg): 145/89\n",
        "            HR (bpm): 80\n",
        "\n",
        "        Note: I made a temporary hack that fixes this particular problem.\n",
        "              We'll see how it shakes out\n",
        "    '''\n",
        "\n",
        "\n",
        "    '''\n",
        "        Separate ALL CAPS lines (Warning... is there ever prose that can be all caps?)\n",
        "    '''\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "    return segments\n",
        "# update these constants to run this script\n",
        "########################################################################################################################\n",
        "OUTPUT_DIR = '/content/drive/MyDrive/Data' # This path will contain tokenized notes. This dir will be the input dir for create_pretrain_data.sh\n",
        "MIMIC_NOTES_FILE = '/content/drive/MyDrive/Data/data75.csv' # This is the path to mimic data if you're reading from a csv. Else uncomment the code to read from the database below\n",
        "\n",
        "# setting sentence boundaries\n",
        "'''\n",
        "def sbd_component(doc):\n",
        "    for i, token in enumerate(doc[:-2]):\n",
        "        # define sentence start if period + titlecase token\n",
        "        if token.text == '.' and doc[i+1].is_title:\n",
        "            doc[i+1].sent_start = True\n",
        "        if token.text == '-' and doc[i+1].text != '-':\n",
        "            doc[i+1].sent_start = True\n",
        "    return doc\n",
        "'''\n",
        "\n",
        "# convert de-identification text into one token\n",
        "def fix_deid_tokens(text, processed_text):\n",
        "    deid_regex  = r\"\\[\\*\\*.{0,15}.*?\\*\\*\\]\"\n",
        "    if text:\n",
        "        indexes = [m.span() for m in re.finditer(deid_regex,text,flags=re.IGNORECASE)]\n",
        "    else:\n",
        "        indexes = []\n",
        "    for start, end in indexes:\n",
        "        processed_text.merge(start_idx=start, end_idx=end)\n",
        "    return processed_text\n",
        "\n",
        "def process_section(section, note, processed_sections):\n",
        "    # perform spacy processing on section\n",
        "    processed_section = nlp(section['sections'])\n",
        "    processed_section = fix_deid_tokens(section['sections'], processed_section)\n",
        "    processed_sections.append(processed_section)\n",
        "\n",
        "def process_note_helper(note):\n",
        "    # split note into sections\n",
        "    note_sections = sent_tokenize_rules(note)\n",
        "    processed_sections = []\n",
        "    section_frame = pd.DataFrame({'sections': note_sections})\n",
        "    section_frame.apply(process_section, args=(note, processed_sections,), axis=1)\n",
        "    return processed_sections\n",
        "\n",
        "def process_text(sent, note):\n",
        "    sent_text = sent['sents'].text\n",
        "    if len(sent_text) > 0 and sent_text.strip() != '\\n':\n",
        "        if '\\n' in sent_text:\n",
        "            sent_text = sent_text.replace('\\n', ' ')\n",
        "        note['text'] += sent_text + '\\n'\n",
        "\n",
        "def get_sentences(processed_section, note):\n",
        "    # get sentences from spacy processing\n",
        "    sent_frame = pd.DataFrame({'sents': list(processed_section['sections'].sents)})\n",
        "    sent_frame.apply(process_text, args=(note,), axis=1)\n",
        "\n",
        "def process_note(note):\n",
        "    try:\n",
        "        note_text = note['text']  # unicode(note['text'])\n",
        "        note['text'] = ''\n",
        "        processed_sections = process_note_helper(note_text)\n",
        "        ps = {'sections': processed_sections}\n",
        "        ps = pd.DataFrame(ps)\n",
        "        ps.apply(get_sentences, args=(note,), axis=1)\n",
        "        return note\n",
        "    except Exception as e:\n",
        "        pass\n",
        "        # print('error', e)\n",
        "\n",
        "\n",
        "start = time.time()\n",
        "tqdm.pandas()\n",
        "\n",
        "print('Begin reading notes')\n",
        "\n",
        "notes = pd.read_csv(MIMIC_NOTES_FILE, index_col=0)\n",
        "\n",
        "print('Number of notes: %d' % len(notes.index))\n",
        "notes['ind'] = list(range(len(notes.index)))\n",
        "\n",
        "# NOTE: `disable=['tagger', 'ner'] was added after paper submission to make this process go faster\n",
        "# our time estimate in the paper did not include the code to skip spacy's NER & tagger\n",
        "nlp = spacy.load('en_core_sci_md', disable=['tagger', 'ner'])\n",
        "# Add  custom sbd_component\n",
        "@spacy.Language.component(\"sbd_component\")\n",
        "def sbd_component(doc):\n",
        "    for i, token in enumerate(doc[:-2]):\n",
        "        # define sentence start if period + titlecase token\n",
        "        if token.text == '.' and doc[i+1].is_title:\n",
        "            doc[i+1].sent_start = True\n",
        "        if token.text == '-' and doc[i+1].text != '-':\n",
        "            doc[i+1].sent_start = True\n",
        "    return doc\n",
        "'''\n",
        "# Add the custom component to the pipeline\n",
        "nlp.add_pipe(\"sbd_component\", before=\"parser\")\n",
        "\n",
        "formatted_notes = notes.progress_apply(process_note, axis=1)\n",
        "with open(OUTPUT_DIR + 'formatted_notes.txt', 'w') as f:\n",
        "    for text in formatted_notes['text']:\n",
        "        if text is not None and len(text) != 0:\n",
        "            f.write(text)\n",
        "            f.write('\\n')\n",
        "\n",
        "end = time.time()\n",
        "print(end - start)\n",
        "print(\"Done formatting notes\")\n",
        "'''"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "vPx4dWgf34JJ",
        "outputId": "929b976a-e62a-4f49-ba04-562aabe19aad"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-3d081070721d>\u001b[0m in \u001b[0;36m<cell line: 371>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    369\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    370\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 371\u001b[0;31m \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    372\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    373\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'time' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nlp.add_pipe(\"sbd_component\", before=\"parser\")\n",
        "\n",
        "formatted_notes = notes.progress_apply(process_note, axis=1)\n",
        "with open(OUTPUT_DIR + 'formatted_notes.txt', 'w') as f:\n",
        "    for text in formatted_notes['text']:\n",
        "        if text is not None and len(text) != 0:\n",
        "            f.write(text)\n",
        "            f.write('\\n')\n",
        "\n",
        "end = time.time()\n",
        "print(end - start)\n",
        "print(\"Done formatting notes\")"
      ],
      "metadata": {
        "id": "qo5_J07WdvMo",
        "outputId": "8f24ef2c-d8fc-456e-80c8-3468cfd4ded9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-fb35d8edadc1>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_pipe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"sbd_component\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbefore\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"parser\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mformatted_notes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnotes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogress_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_note\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mOUTPUT_DIR\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'formatted_notes.txt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mtext\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mformatted_notes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'nlp' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(notes.head(3))"
      ],
      "metadata": {
        "id": "kfUlT9r3TtTY",
        "outputId": "3536519d-22ab-49cc-a2ad-ea19d7a1e7ed",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                                         TEXT  \\\n",
            "SUBJECT_ID                                                      \n",
            "52247       Admission Date:  [**2198-7-4**]              D...   \n",
            "57181       Admission Date:  [**2171-10-24**]             ...   \n",
            "26896       Admission Date:  [**2129-3-22**]              ...   \n",
            "\n",
            "                     CATEGORY  ind  \n",
            "SUBJECT_ID                          \n",
            "52247       Discharge summary    0  \n",
            "57181       Discharge summary    1  \n",
            "26896       Discharge summary    2  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(formatted_notes.head(3))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hRBcobQURbbV",
        "outputId": "26e8e513-17aa-420f-a6b1-a3f2867de9d1"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "SUBJECT_ID\n",
            "52247    None\n",
            "57181    None\n",
            "26896    None\n",
            "dtype: object\n"
          ]
        }
      ]
    }
  ]
}