FP_FN_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, f1_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from transformers import (\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    "    AutoModel,\n",
    "    AutoTokenizer,\n",
    "    AutoModelForMaskedLM,\n",
    "    AutoModelForSequenceClassification,\n",
    ")\n",
    "import json\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "encode = spacy.load(\"en_core_web_lg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "negative_example_ids = []\n",
    "negative_example_json = json.load(open(\"negative_examples_100.json\"))\n",
    "for example in negative_example_json:\n",
    "    negative_example_ids.append((example[\"ap\"], example[\"technique\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"w_dict.json\", \"r\") as f:\n",
    "    w_dict = json.load(f)\n",
    "with open(\"ap_dict.json\", \"r\") as f:\n",
    "    ap_dict = json.load(f)\n",
    "with open(\"technique_dict.json\", \"r\") as f:\n",
    "    technique_dict = json.load(f)\n",
    "with open(\"tactic_dict.json\", \"r\") as f:\n",
    "    tactic_dict = json.load(f)\n",
    "with open(\"cwe_names.json\", \"r\") as f:\n",
    "    cwe_names = json.load(f)\n",
    "with open(\"ap_names.json\", \"r\") as f:\n",
    "    ap_names = json.load(f)\n",
    "with open(\"technique_names.json\", \"r\") as f:\n",
    "    technique_names = json.load(f)\n",
    "with open(\"tactic_names.json\", \"r\") as f:\n",
    "    tactic_names = json.load(f)\n",
    "with open(\"ap_mitigation_descriptions.json\", \"r\") as f:\n",
    "    ap_mitigation_descriptions = json.load(f)\n",
    "with open(\"cwe_mitigation_descriptions.json\", \"r\") as f:\n",
    "    cwe_mitigation_descriptions = json.load(f)\n",
    "with open(\"tech_mitigation_names.json\", \"r\") as f:\n",
    "    tech_mitigation_names = json.load(f)\n",
    "with open(\"tech_detection_names.json\", \"r\") as f:\n",
    "    tech_detection_names = json.load(f)\n",
    "with open(\"ap_detection_descriptions.json\", \"r\") as f:\n",
    "    ap_detection_descriptions = json.load(f)\n",
    "with open(\"cwe_detection_descriptions.json\", \"r\") as f:\n",
    "    cwe_detection_descriptions = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"cwe_mitigation_ids_temp.json\")\n",
    "w_mitigation = json.load(f)\n",
    "\n",
    "f = open(\"capec_mitigation_temp.json\")\n",
    "ap_mitigation = json.load(f)\n",
    "\n",
    "f = open(\"technique_mitigation_temp.json\")\n",
    "technique_mitigation = json.load(f)\n",
    "\n",
    "f = open(\"technique_detection_temp.json\")\n",
    "technique_detection = json.load(f)\n",
    "\n",
    "f = open(\"capec_detection_temp.json\")\n",
    "ap_detection = json.load(f)\n",
    "\n",
    "f = open(\"cwe_detection_temp.json\")\n",
    "w_detection = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_example_ids = []\n",
    "for ap in ap_dict:\n",
    "    for technique in ap_dict[ap][\"techniques\"]:\n",
    "        positive_example_ids.append((ap, technique))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_ids = positive_example_ids + negative_example_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer()"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ap_name_vectorizer = CountVectorizer()\n",
    "ap_name_vectorizer.fit(ap_names)\n",
    "\n",
    "technique_name_vectorizer = CountVectorizer()\n",
    "technique_name_vectorizer.fit(technique_names)\n",
    "\n",
    "cwe_name_vectorizer = CountVectorizer()\n",
    "cwe_name_vectorizer.fit(cwe_names)\n",
    "\n",
    "tactic_name_vectorizer = CountVectorizer()\n",
    "tactic_name_vectorizer.fit(tactic_names)\n",
    "\n",
    "ap_mitigation_vectorizer = CountVectorizer()\n",
    "cwe_mitigation_vectorizer = CountVectorizer()\n",
    "tech_mitigation_vectorizer = CountVectorizer()\n",
    "\n",
    "ap_mitigation_vectorizer.fit(ap_mitigation_descriptions)\n",
    "cwe_mitigation_vectorizer.fit(cwe_mitigation_descriptions)\n",
    "tech_mitigation_vectorizer.fit(tech_mitigation_names)\n",
    "\n",
    "ap_detection_vectorizer = CountVectorizer()\n",
    "cwe_detection_vectorizer = CountVectorizer()\n",
    "tech_detection_vectorizer = CountVectorizer()\n",
    "\n",
    "ap_detection_vectorizer.fit(ap_detection_descriptions)\n",
    "cwe_detection_vectorizer.fit(cwe_detection_descriptions)\n",
    "tech_detection_vectorizer.fit(tech_detection_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "device = \"cpu\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "pretrained_model = AutoModel.from_pretrained(\"bert-base-uncased\").to(device)\n",
    "\n",
    "model_path = \"bert_base\"\n",
    "finetuned_model = AutoModelForMaskedLM.from_pretrained(model_path).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vector_encoding(\n",
    "    encoding_type, text, vectorizer=None, bert_output_type=None, bert_finetuned=False\n",
    "):\n",
    "    if encoding_type == \"None\":\n",
    "        return text\n",
    "    elif encoding_type == \"BoW\":\n",
    "        return vectorizer_transform(text, vectorizer)\n",
    "    elif encoding_type == \"spaCy\":\n",
    "        return spaCy_vector(text)\n",
    "    elif encoding_type == \"BERT\":\n",
    "        if bert_finetuned:\n",
    "            model = finetuned_model\n",
    "        else:\n",
    "            model = pretrained_model\n",
    "\n",
    "        if bert_output_type == \"pooler_output\":\n",
    "            return get_pooler_output(model, text)\n",
    "        elif bert_output_type == \"hidden_state\":\n",
    "            return get_hidden_state(model, text)\n",
    "\n",
    "\n",
    "def vectorizer_transform(input_to_BoW, vectorizer):\n",
    "    return vectorizer.transform([input_to_BoW])[0].toarray().flatten()\n",
    "\n",
    "\n",
    "def spaCy_vector(text):\n",
    "    return encode(text).vector\n",
    "\n",
    "\n",
    "def get_pooler_output(model, text):\n",
    "    inputs = tokenizer(text.lower(), truncation=True, return_tensors=\"pt\").to(device)\n",
    "    outputs = model(**inputs)\n",
    "    pooled_output = outputs.pooler_output\n",
    "    return pooled_output.detach().cpu().numpy().flatten()\n",
    "\n",
    "\n",
    "def get_hidden_state(model, text):\n",
    "    inputs = tokenizer(text.lower(), truncation=True, return_tensors=\"pt\").to(device)\n",
    "    outputs = model(**inputs, output_hidden_states=True)\n",
    "    hidden_states = outputs.hidden_states\n",
    "    return hidden_states[-1][:, 0, :].detach().cpu().numpy().flatten()\n",
    "\n",
    "\n",
    "def append_data(\n",
    "    encoding_type,\n",
    "    data_combo,\n",
    "    ap,\n",
    "    technique,\n",
    "    bert_output_type=None,\n",
    "    bert_finetuned=False,\n",
    "):\n",
    "    output = []\n",
    "    vectorizer = CountVectorizer()\n",
    "    if data_combo == \"A0\":\n",
    "        vectorizer.fit(ap_names + technique_names)\n",
    "\n",
    "    elif data_combo == \"A1\":\n",
    "        vectorizer.fit(ap_names + technique_names + cwe_names + tactic_names)\n",
    "\n",
    "    elif data_combo == \"A1 + MI\":\n",
    "        vectorizer.fit(\n",
    "            ap_names\n",
    "            + technique_names\n",
    "            + cwe_names\n",
    "            + tactic_names\n",
    "            + cwe_mitigation_descriptions\n",
    "            + ap_mitigation_descriptions\n",
    "            + tech_mitigation_names\n",
    "        )\n",
    "\n",
    "    elif data_combo == \"A1 + D\":\n",
    "        vectorizer.fit(\n",
    "            ap_names\n",
    "            + technique_names\n",
    "            + cwe_names\n",
    "            + tactic_names\n",
    "            + cwe_detection_descriptions\n",
    "            + ap_detection_descriptions\n",
    "            + tech_detection_names\n",
    "        )\n",
    "\n",
    "    elif data_combo == \"A1 + MI + D\":\n",
    "        vectorizer.fit(\n",
    "            ap_names\n",
    "            + technique_names\n",
    "            + cwe_names\n",
    "            + tactic_names\n",
    "            + cwe_mitigation_descriptions\n",
    "            + ap_mitigation_descriptions\n",
    "            + tech_mitigation_names\n",
    "            + cwe_detection_descriptions\n",
    "            + ap_detection_descriptions\n",
    "            + tech_detection_names\n",
    "        )\n",
    "\n",
    "    output.append(ap_dict[ap][\"name\"])\n",
    "    output.append(technique_dict[technique][\"name\"])\n",
    "\n",
    "    if \"A1\" in data_combo:\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            output.append(w_dict[cwe][\"name\"])\n",
    "\n",
    "        for tac in technique_dict[technique][\"tactics\"]:\n",
    "            output.append(tactic_dict[tac][\"name\"])\n",
    "\n",
    "    if data_combo in [\"A1 + MI\", \"A1 + MI + D\"]:\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            for mitigation in w_dict[cwe][\"mitigations\"]:\n",
    "                for cwe_mit in w_mitigation:\n",
    "                    if mitigation == cwe_mit[\"_id\"]:\n",
    "                        output.append(cwe_mit[\"metadata\"][\"Description\"])\n",
    "\n",
    "        for mitigation in ap_dict[ap][\"mitigations\"]:\n",
    "            for ap_mit in ap_mitigation:\n",
    "                if mitigation == ap_mit[\"_id\"]:\n",
    "                    output.append(ap_mit[\"metadata\"])\n",
    "\n",
    "        for mitigation in technique_dict[technique][\"mitigations\"]:\n",
    "            for tech_mit in technique_mitigation:\n",
    "                if mitigation == tech_mit[\"_id\"]:\n",
    "                    output.append(tech_mit[\"name\"])\n",
    "\n",
    "    if data_combo in [\"A1 + D\", \"A1 + MI + D\"]:\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            for detection in w_dict[cwe][\"detections\"]:\n",
    "                for cwe_det in w_detection:\n",
    "                    if detection == cwe_det[\"_id\"]:\n",
    "                        output.append(cwe_det[\"metadata\"][\"Description\"])\n",
    "\n",
    "        for detection in ap_dict[ap][\"detections\"]:\n",
    "            for ap_det in ap_detection:\n",
    "                if detection == ap_det[\"_id\"]:\n",
    "                    output.append(ap_det[\"metadata\"])\n",
    "\n",
    "        for detection in technique_dict[technique][\"detections\"]:\n",
    "            for tech_det in technique_mitigation:\n",
    "                if detection == tech_det[\"_id\"]:\n",
    "                    output.append(tech_det[\"metadata\"])\n",
    "\n",
    "    output = \" \".join(output)\n",
    "    return vector_encoding(\n",
    "        encoding_type, output, vectorizer, bert_output_type, bert_finetuned\n",
    "    )\n",
    "\n",
    "\n",
    "def handle_data(\n",
    "    encoding_type,\n",
    "    data_combo,\n",
    "    ap,\n",
    "    technique,\n",
    "    bert_output_type=None,\n",
    "    bert_finetuned=False,\n",
    "):\n",
    "    example = []\n",
    "    try:\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                ap_dict[ap][\"name\"],\n",
    "                ap_name_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                technique_dict[technique][\"name\"],\n",
    "                technique_name_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "    except KeyError as e:\n",
    "        print(f\"Error {e}\")\n",
    "        return []\n",
    "\n",
    "    if \"A1\" in data_combo:\n",
    "        tactics = []\n",
    "        for tac in technique_dict[technique][\"tactics\"]:\n",
    "            tactics.append(tactic_dict[tac][\"name\"])\n",
    "        tactics = \" \".join(tactics)\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                tactics,\n",
    "                tactic_name_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "\n",
    "        cwes = []\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            cwes.append(w_dict[cwe][\"name\"])\n",
    "        cwes = \" \".join(cwes)\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                cwes,\n",
    "                cwe_name_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "\n",
    "    if data_combo in [\"A1 + MI\", \"A1 + MI + D\"]:\n",
    "        cwe_mitigations = []\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            for mitigation in w_dict[cwe][\"mitigations\"]:\n",
    "                for cwe_mit in w_mitigation:\n",
    "                    if mitigation == cwe_mit[\"_id\"]:\n",
    "                        cwe_mitigations.append(cwe_mit[\"metadata\"][\"Description\"])\n",
    "        cwe_mitigations = \" \".join(cwe_mitigations)\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                cwe_mitigations,\n",
    "                cwe_mitigation_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "\n",
    "        capec_mitigations = []\n",
    "        for mitigation in ap_dict[ap][\"mitigations\"]:\n",
    "            for ap_mit in ap_mitigation:\n",
    "                if mitigation == ap_mit[\"_id\"]:\n",
    "                    capec_mitigations.append(ap_mit[\"metadata\"])\n",
    "        capec_mitigations = \" \".join(capec_mitigations)\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                capec_mitigations,\n",
    "                ap_mitigation_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "\n",
    "        tech_mitigations = []\n",
    "        for mitigation in technique_dict[technique][\"mitigations\"]:\n",
    "            for tech_mit in technique_mitigation:\n",
    "                if mitigation == tech_mit[\"_id\"]:\n",
    "                    tech_mitigations.append(tech_mit[\"name\"])\n",
    "        tech_mitigations = \" \".join(tech_mitigations)\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                tech_mitigations,\n",
    "                tech_mitigation_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "\n",
    "    if data_combo in [\"A1 + D\", \"A1 + MI + D\"]:\n",
    "        cwe_detections = []\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            for detection in w_dict[cwe][\"detections\"]:\n",
    "                for cwe_det in w_detection:\n",
    "                    if detection == cwe_det[\"_id\"]:\n",
    "                        cwe_detections.append(cwe_det[\"metadata\"][\"Description\"])\n",
    "\n",
    "        cwe_detections = \" \".join(cwe_detections)\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                cwe_detections,\n",
    "                cwe_detection_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "\n",
    "        capec_detections = []\n",
    "        for detection in ap_dict[ap][\"detections\"]:\n",
    "            for ap_det in ap_detection:\n",
    "                if detection == ap_det[\"_id\"]:\n",
    "                    capec_detections.append(ap_det[\"metadata\"])\n",
    "        capec_detections = \" \".join(capec_detections)\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                capec_detections,\n",
    "                ap_detection_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "\n",
    "        tech_detections = []\n",
    "        for detection in technique_dict[technique][\"detections\"]:\n",
    "            for tech_det in technique_detection:\n",
    "                if detection == tech_det[\"_id\"]:\n",
    "                    tech_detections.append(tech_det[\"name\"])\n",
    "        tech_detections = \" \".join(tech_detections)\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                tech_detections,\n",
    "                tech_detection_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "\n",
    "    return np.hstack(example)\n",
    "\n",
    "\n",
    "def encode_data(\n",
    "    encoding_type,\n",
    "    data_combo,\n",
    "    ap,\n",
    "    technique,\n",
    "    bert_output_type=None,\n",
    "    bert_finetuned=False,\n",
    "):\n",
    "    example = []\n",
    "\n",
    "    try:\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                ap_dict[ap][\"name\"],\n",
    "                ap_name_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "        example.append(\n",
    "            vector_encoding(\n",
    "                encoding_type,\n",
    "                technique_dict[technique][\"name\"],\n",
    "                technique_name_vectorizer,\n",
    "                bert_output_type,\n",
    "                bert_finetuned,\n",
    "            )\n",
    "        )\n",
    "    except KeyError as e:\n",
    "        print(f\"Error {e}\")\n",
    "        return []\n",
    "\n",
    "    if \"A1\" in data_combo:\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            example.append(\n",
    "                vector_encoding(\n",
    "                    encoding_type,\n",
    "                    w_dict[cwe][\"name\"],\n",
    "                    cwe_name_vectorizer,\n",
    "                    bert_output_type,\n",
    "                    bert_finetuned,\n",
    "                )\n",
    "            )\n",
    "\n",
    "        for tac in technique_dict[technique][\"tactics\"]:\n",
    "            example.append(\n",
    "                vector_encoding(\n",
    "                    encoding_type,\n",
    "                    tactic_dict[tac][\"name\"],\n",
    "                    tactic_name_vectorizer,\n",
    "                    bert_output_type,\n",
    "                    bert_finetuned,\n",
    "                )\n",
    "            )\n",
    "\n",
    "    if data_combo in [\"A1 + MI\", \"A1 + MI + D\"]:\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            for mitigation in w_dict[cwe][\"mitigations\"]:\n",
    "                for cwe_mit in w_mitigation:\n",
    "                    if mitigation == cwe_mit[\"_id\"]:\n",
    "                        example.append(\n",
    "                            vector_encoding(\n",
    "                                encoding_type,\n",
    "                                cwe_mit[\"metadata\"][\"Description\"],\n",
    "                                cwe_mitigation_vectorizer,\n",
    "                                bert_output_type,\n",
    "                                bert_finetuned,\n",
    "                            )\n",
    "                        )\n",
    "\n",
    "        for mitigation in ap_dict[ap][\"mitigations\"]:\n",
    "            for ap_mit in ap_mitigation:\n",
    "                if mitigation == ap_mit[\"_id\"]:\n",
    "                    example.append(\n",
    "                        vector_encoding(\n",
    "                            encoding_type,\n",
    "                            ap_mit[\"metadata\"],\n",
    "                            ap_mitigation_vectorizer,\n",
    "                            bert_output_type,\n",
    "                            bert_finetuned,\n",
    "                        )\n",
    "                    )\n",
    "\n",
    "        for mitigation in technique_dict[technique][\"mitigations\"]:\n",
    "            for tech_mit in technique_mitigation:\n",
    "                if mitigation == tech_mit[\"_id\"]:\n",
    "                    example.append(\n",
    "                        vector_encoding(\n",
    "                            encoding_type,\n",
    "                            tech_mit[\"name\"],\n",
    "                            tech_mitigation_vectorizer,\n",
    "                            bert_output_type,\n",
    "                            bert_finetuned,\n",
    "                        )\n",
    "                    )\n",
    "\n",
    "    if data_combo in [\"A1 + D\", \"A1 + MI + D\"]:\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            for detection in w_dict[cwe][\"detections\"]:\n",
    "                for cwe_det in w_detection:\n",
    "                    if detection == cwe_det[\"_id\"]:\n",
    "                        example.append(\n",
    "                            vector_encoding(\n",
    "                                encoding_type,\n",
    "                                cwe_det[\"metadata\"][\"Description\"],\n",
    "                                cwe_detection_vectorizer,\n",
    "                                bert_output_type,\n",
    "                                bert_finetuned,\n",
    "                            )\n",
    "                        )\n",
    "\n",
    "        for detection in ap_dict[ap][\"detections\"]:\n",
    "            for ap_det in ap_detection:\n",
    "                if detection == ap_det[\"_id\"]:\n",
    "                    example.append(\n",
    "                        vector_encoding(\n",
    "                            encoding_type,\n",
    "                            ap_det[\"metadata\"],\n",
    "                            ap_detection_vectorizer,\n",
    "                            bert_output_type,\n",
    "                            bert_finetuned,\n",
    "                        )\n",
    "                    )\n",
    "\n",
    "        for detection in technique_dict[technique][\"detections\"]:\n",
    "            for tech_det in technique_detection:\n",
    "                if detection == tech_det[\"_id\"]:\n",
    "                    example.append(\n",
    "                        vector_encoding(\n",
    "                            encoding_type,\n",
    "                            tech_det[\"name\"],\n",
    "                            tech_detection_vectorizer,\n",
    "                            bert_output_type,\n",
    "                            bert_finetuned,\n",
    "                        )\n",
    "                    )\n",
    "\n",
    "    return np.hstack(example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_classifications(name: str):\n",
    "    print(f\"Classify {name}\")\n",
    "    examples = []\n",
    "    labels = []\n",
    "    for i, (ap, technique) in enumerate(example_ids):\n",
    "        if name == \"BERT\":\n",
    "            _example = handle_data(\n",
    "                \"BERT\",\n",
    "                \"A1 + MI\",\n",
    "                ap,\n",
    "                technique,\n",
    "                bert_output_type=\"hidden_state\",\n",
    "                bert_finetuned=True,\n",
    "            )\n",
    "        elif name == \"BoW\":\n",
    "            _example = handle_data(\"BoW\", \"A1 + MI\", ap, technique)\n",
    "        elif name == \"spaCy\":\n",
    "            _example = handle_data(\"spaCy\", \"A1 + MI\", ap, technique)\n",
    "\n",
    "        if len(_example) == 0:\n",
    "            continue\n",
    "\n",
    "        examples.append(_example)\n",
    "        if i < len(example_ids) / 2:\n",
    "            labels.append(1)\n",
    "        else:\n",
    "            labels.append(0)\n",
    "\n",
    "        if i > 1:\n",
    "            assert len(_example) == len(examples[-2])\n",
    "\n",
    "    X_train, X_test, y_train, y_test = train_test_split(\n",
    "        examples, labels, test_size=0.3, random_state=0\n",
    "    )\n",
    "\n",
    "    clf = RandomForestClassifier(random_state=0)\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "\n",
    "    FPs = []\n",
    "    FNs = []\n",
    "\n",
    "    test_example_ids = {}\n",
    "\n",
    "    for i in range(len(X_test)):\n",
    "        for j in range(len(examples)):\n",
    "            if (X_test[i] == examples[j]).all():\n",
    "                test_example_ids[i] = example_ids[j]\n",
    "                break\n",
    "\n",
    "    for i in range(len(y_pred)):\n",
    "        if y_pred[i] == 1 and y_test[i] != y_pred[i]:\n",
    "            FPs.append(test_example_ids[i])\n",
    "        if y_pred[i] == 0 and y_test[i] != y_pred[i]:\n",
    "            FNs.append(test_example_ids[i])\n",
    "\n",
    "    with open(f\"Best_{name}_FP_ids.json\", \"w\") as f:\n",
    "        json.dump(FPs, f)\n",
    "\n",
    "    with open(f\"Best_{name}_FN_ids.json\", \"w\") as f:\n",
    "        json.dump(FNs, f)\n",
    "\n",
    "    confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classify BoW\n",
      "Error 'capec/capec_01246'\n",
      "Error 'capec/capec_01137'\n",
      "Error 'capec/capec_01211'\n",
      "Error 'capec/capec_01320'\n",
      "Error 'capec/capec_01239'\n",
      "Error 'technique/technique_00648'\n",
      "Error 'capec/capec_01296'\n",
      "Error 'technique/technique_00640'\n",
      "Error 'capec/capec_01211'\n",
      "Error 'capec/capec_01256'\n",
      "Error 'capec/capec_01131'\n",
      "Error 'technique/technique_00641'\n",
      "Error 'capec/capec_01228'\n",
      "Error 'capec/capec_01304'\n",
      "Error 'capec/capec_01279'\n",
      "Error 'capec/capec_01224'\n",
      "Error 'capec/capec_01283'\n",
      "Error 'capec/capec_01276'\n",
      "Error 'capec/capec_01283'\n",
      "Error 'technique/technique_00625'\n",
      "Error 'capec/capec_01312'\n",
      "Error 'technique/technique_00642'\n",
      "Error 'capec/capec_01276'\n",
      "Error 'capec/capec_01152'\n",
      "Error 'capec/capec_01227'\n",
      "Error 'technique/technique_00642'\n",
      "Error 'capec/capec_01159'\n",
      "Error 'capec/capec_01310'\n",
      "Error 'capec/capec_01270'\n",
      "Error 'technique/technique_00665'\n",
      "Error 'technique/technique_00635'\n",
      "Error 'capec/capec_01312'\n",
      "Error 'technique/technique_00635'\n",
      "Error 'capec/capec_01213'\n",
      "Error 'technique/technique_00710'\n",
      "Error 'capec/capec_01158'\n",
      "Error 'technique/technique_00677'\n",
      "Error 'capec/capec_01169'\n",
      "Error 'capec/capec_01201'\n",
      "Error 'capec/capec_01306'\n",
      "Error 'capec/capec_01175'\n",
      "Error 'capec/capec_01205'\n",
      "Error 'capec/capec_01142'\n",
      "Error 'capec/capec_01243'\n",
      "Error 'capec/capec_01297'\n",
      "Error 'technique/technique_00639'\n",
      "Error 'technique/technique_00704'\n",
      "Error 'technique/technique_00619'\n",
      "Error 'technique/technique_00667'\n",
      "Error 'capec/capec_01181'\n",
      "Error 'capec/capec_01304'\n",
      "Error 'capec/capec_01304'\n",
      "Error 'capec/capec_01179'\n",
      "Error 'capec/capec_01180'\n",
      "Error 'capec/capec_01221'\n",
      "Error 'capec/capec_01229'\n",
      "Error 'capec/capec_01150'\n",
      "Error 'capec/capec_01264'\n",
      "Error 'technique/technique_00637'\n",
      "Error 'technique/technique_00720'\n",
      "Error 'capec/capec_01210'\n",
      "Error 'technique/technique_00684'\n",
      "Error 'capec/capec_01256'\n",
      "Error 'technique/technique_00586'\n",
      "Classify spaCy\n",
      "Error 'capec/capec_01246'\n",
      "Error 'capec/capec_01137'\n",
      "Error 'capec/capec_01211'\n",
      "Error 'capec/capec_01320'\n",
      "Error 'capec/capec_01239'\n",
      "Error 'technique/technique_00648'\n",
      "Error 'capec/capec_01296'\n",
      "Error 'technique/technique_00640'\n",
      "Error 'capec/capec_01211'\n",
      "Error 'capec/capec_01256'\n",
      "Error 'capec/capec_01131'\n",
      "Error 'technique/technique_00641'\n",
      "Error 'capec/capec_01228'\n",
      "Error 'capec/capec_01304'\n",
      "Error 'capec/capec_01279'\n",
      "Error 'capec/capec_01224'\n",
      "Error 'capec/capec_01283'\n",
      "Error 'capec/capec_01276'\n",
      "Error 'capec/capec_01283'\n",
      "Error 'technique/technique_00625'\n",
      "Error 'capec/capec_01312'\n",
      "Error 'technique/technique_00642'\n",
      "Error 'capec/capec_01276'\n",
      "Error 'capec/capec_01152'\n",
      "Error 'capec/capec_01227'\n",
      "Error 'technique/technique_00642'\n",
      "Error 'capec/capec_01159'\n",
      "Error 'capec/capec_01310'\n",
      "Error 'capec/capec_01270'\n",
      "Error 'technique/technique_00665'\n",
      "Error 'technique/technique_00635'\n",
      "Error 'capec/capec_01312'\n",
      "Error 'technique/technique_00635'\n",
      "Error 'capec/capec_01213'\n",
      "Error 'technique/technique_00710'\n",
      "Error 'capec/capec_01158'\n",
      "Error 'technique/technique_00677'\n",
      "Error 'capec/capec_01169'\n",
      "Error 'capec/capec_01201'\n",
      "Error 'capec/capec_01306'\n",
      "Error 'capec/capec_01175'\n",
      "Error 'capec/capec_01205'\n",
      "Error 'capec/capec_01142'\n",
      "Error 'capec/capec_01243'\n",
      "Error 'capec/capec_01297'\n",
      "Error 'technique/technique_00639'\n",
      "Error 'technique/technique_00704'\n",
      "Error 'technique/technique_00619'\n",
      "Error 'technique/technique_00667'\n",
      "Error 'capec/capec_01181'\n",
      "Error 'capec/capec_01304'\n",
      "Error 'capec/capec_01304'\n",
      "Error 'capec/capec_01179'\n",
      "Error 'capec/capec_01180'\n",
      "Error 'capec/capec_01221'\n",
      "Error 'capec/capec_01229'\n",
      "Error 'capec/capec_01150'\n",
      "Error 'capec/capec_01264'\n",
      "Error 'technique/technique_00637'\n",
      "Error 'technique/technique_00720'\n",
      "Error 'capec/capec_01210'\n",
      "Error 'technique/technique_00684'\n",
      "Error 'capec/capec_01256'\n",
      "Error 'technique/technique_00586'\n",
      "Classify BERT\n",
      "Error 'capec/capec_01246'\n",
      "Error 'capec/capec_01137'\n",
      "Error 'capec/capec_01211'\n",
      "Error 'capec/capec_01320'\n",
      "Error 'capec/capec_01239'\n",
      "Error 'technique/technique_00648'\n",
      "Error 'capec/capec_01296'\n",
      "Error 'technique/technique_00640'\n",
      "Error 'capec/capec_01211'\n",
      "Error 'capec/capec_01256'\n",
      "Error 'capec/capec_01131'\n",
      "Error 'technique/technique_00641'\n",
      "Error 'capec/capec_01228'\n",
      "Error 'capec/capec_01304'\n",
      "Error 'capec/capec_01279'\n",
      "Error 'capec/capec_01224'\n",
      "Error 'capec/capec_01283'\n",
      "Error 'capec/capec_01276'\n",
      "Error 'capec/capec_01283'\n",
      "Error 'technique/technique_00625'\n",
      "Error 'capec/capec_01312'\n",
      "Error 'technique/technique_00642'\n",
      "Error 'capec/capec_01276'\n",
      "Error 'capec/capec_01152'\n",
      "Error 'capec/capec_01227'\n",
      "Error 'technique/technique_00642'\n",
      "Error 'capec/capec_01159'\n",
      "Error 'capec/capec_01310'\n",
      "Error 'capec/capec_01270'\n",
      "Error 'technique/technique_00665'\n",
      "Error 'technique/technique_00635'\n",
      "Error 'capec/capec_01312'\n",
      "Error 'technique/technique_00635'\n",
      "Error 'capec/capec_01213'\n",
      "Error 'technique/technique_00710'\n",
      "Error 'capec/capec_01158'\n",
      "Error 'technique/technique_00677'\n",
      "Error 'capec/capec_01169'\n",
      "Error 'capec/capec_01201'\n",
      "Error 'capec/capec_01306'\n",
      "Error 'capec/capec_01175'\n",
      "Error 'capec/capec_01205'\n",
      "Error 'capec/capec_01142'\n",
      "Error 'capec/capec_01243'\n",
      "Error 'capec/capec_01297'\n",
      "Error 'technique/technique_00639'\n",
      "Error 'technique/technique_00704'\n",
      "Error 'technique/technique_00619'\n",
      "Error 'technique/technique_00667'\n",
      "Error 'capec/capec_01181'\n",
      "Error 'capec/capec_01304'\n",
      "Error 'capec/capec_01304'\n",
      "Error 'capec/capec_01179'\n",
      "Error 'capec/capec_01180'\n",
      "Error 'capec/capec_01221'\n",
      "Error 'capec/capec_01229'\n",
      "Error 'capec/capec_01150'\n",
      "Error 'capec/capec_01264'\n",
      "Error 'technique/technique_00637'\n",
      "Error 'technique/technique_00720'\n",
      "Error 'capec/capec_01210'\n",
      "Error 'technique/technique_00684'\n",
      "Error 'capec/capec_01256'\n",
      "Error 'technique/technique_00586'\n"
     ]
    }
   ],
   "source": [
    "for name in (\"BoW\", \"spaCy\", \"BERT\"):\n",
    "    get_classifications(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gather_text_data(ap, technique, data_combo):\n",
    "    output = {}\n",
    "    output[\"AP\"] = ap_dict[ap][\"name\"]\n",
    "    output[\"Technique\"] = technique_dict[technique][\"name\"]\n",
    "\n",
    "    if \"A1\" in data_combo:\n",
    "        output[\"CWEs\"] = []\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            output[\"CWEs\"].append(w_dict[cwe][\"name\"])\n",
    "        output[\"Tactics\"] = []\n",
    "        for tac in technique_dict[technique][\"tactics\"]:\n",
    "            output[\"Tactics\"].append(tactic_dict[tac][\"name\"])\n",
    "\n",
    "    if \"A1 + MI\" in data_combo:\n",
    "        output[\"CWE Mitigations\"] = []\n",
    "        for cwe in ap_dict[ap][\"cwes\"]:\n",
    "            for mitigation in w_dict[cwe][\"mitigations\"]:\n",
    "                for cwe_mit in w_mitigation:\n",
    "                    if mitigation == cwe_mit[\"_id\"]:\n",
    "                        output[\"CWE Mitigations\"].append(\n",
    "                            cwe_mit[\"metadata\"][\"Description\"]\n",
    "                        )\n",
    "\n",
    "        output[\"AP Mitigations\"] = []\n",
    "        for mitigation in ap_dict[ap][\"mitigations\"]:\n",
    "            for ap_mit in ap_mitigation:\n",
    "                if mitigation == ap_mit[\"_id\"]:\n",
    "                    output[\"AP Mitigations\"].append(ap_mit[\"metadata\"])\n",
    "\n",
    "        output[\"Technique Mitigations\"] = []\n",
    "        for mitigation in technique_dict[technique][\"mitigations\"]:\n",
    "            for tech_mit in technique_mitigation:\n",
    "                if mitigation == tech_mit[\"_id\"]:\n",
    "                    output[\"Technique Mitigations\"].append(tech_mit[\"name\"])\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"Best_BoW_FN_ids.json\")\n",
    "BoW_FN = json.load(f)\n",
    "f = open(\"Best_spaCy_FN_ids.json\")\n",
    "spaCy_FN = json.load(f)\n",
    "f = open(\"Best_BERT_FN_ids.json\")\n",
    "bert_FN = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BoW FNs  7\n",
      "spaCy FNs  7\n",
      "bert FNs  7\n"
     ]
    }
   ],
   "source": [
    "BoW_FN = set(tuple(x) for x in BoW_FN)\n",
    "spaCy_FN = set(tuple(x) for x in spaCy_FN)\n",
    "bert_FN = set(tuple(x) for x in bert_FN)\n",
    "\n",
    "print(\"BoW FNs \", len(BoW_FN))\n",
    "print(\"spaCy FNs \", len(spaCy_FN))\n",
    "print(\"bert FNs \", len(bert_FN))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_three_encodings = BoW_FN & spaCy_FN & bert_FN\n",
    "len(all_three_encodings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BoW_and_spaCy = (BoW_FN & spaCy_FN) - bert_FN\n",
    "len(BoW_and_spaCy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BoW_and_bert = (BoW_FN & bert_FN) - spaCy_FN\n",
    "len(BoW_and_bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spaCy_and_bert = (spaCy_FN & bert_FN) - BoW_FN\n",
    "len(spaCy_and_bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_BoW = BoW_FN - spaCy_FN - bert_FN\n",
    "len(only_BoW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_spaCy = spaCy_FN - BoW_FN - bert_FN\n",
    "len(only_spaCy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_bert = bert_FN - BoW_FN - spaCy_FN\n",
    "len(only_bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BoW FPs  11\n",
      "spaCy FPs  11\n",
      "bert FPs  12\n"
     ]
    }
   ],
   "source": [
    "f = open(\"Best_BoW_FP_ids.json\")\n",
    "BoW_FP = json.load(f)\n",
    "f = open(\"Best_spaCy_FP_ids.json\")\n",
    "spaCy_FP = json.load(f)\n",
    "f = open(\"Best_BERT_FP_ids.json\")\n",
    "bert_FP = json.load(f)\n",
    "\n",
    "BoW_FP = set(tuple(x) for x in BoW_FP)\n",
    "spaCy_FP = set(tuple(x) for x in spaCy_FP)\n",
    "bert_FP = set(tuple(x) for x in bert_FP)\n",
    "\n",
    "print(\"BoW FPs \", len(BoW_FP))\n",
    "print(\"spaCy FPs \", len(spaCy_FP))\n",
    "print(\"bert FPs \", len(bert_FP))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_three_encodings = BoW_FP & spaCy_FP & bert_FP\n",
    "len(all_three_encodings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BoW_and_spaCy = (BoW_FN & spaCy_FN) - bert_FN\n",
    "len(BoW_and_spaCy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BoW_and_bert = (BoW_FN & bert_FN) - spaCy_FN\n",
    "len(BoW_and_bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spaCy_and_bert = (spaCy_FN & bert_FN) - BoW_FN\n",
    "len(spaCy_and_bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_BoW = BoW_FN - spaCy_FN - bert_FN\n",
    "len(only_BoW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_spaCy = spaCy_FN - BoW_FN - bert_FN\n",
    "len(only_spaCy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_bert = bert_FN - BoW_FN - spaCy_FN\n",
    "len(only_bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all negative examples\n",
    "all_negative_examples = []\n",
    "for ap in ap_dict:\n",
    "    for technique in technique_dict:\n",
    "        if technique not in ap_dict[ap][\"techniques\"]:\n",
    "            all_negative_examples.append((ap, technique))\n",
    "\n",
    "all_negative_examples = list(set(all_negative_examples) - set(example_ids))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "fe5398c8201012d3ec3837fc46b21f6bb5cc60fe8bdaa4e264ad4d58fc1758ab"
  },
  "kernelspec": {
   "display_name": "Python 3.6.9 ('venv_BRON_ML')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}