final_generate_gazetteers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"alert alert-success\">\n",
    "Jump to the final section for the generation of gazetteers from prefetched Wikidata queries\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import json\n",
    "import pickle \n",
    "\n",
    "import requests\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "from SPARQLWrapper import SPARQLWrapper, JSON"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get all subclasses labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = \"\"\n",
    "sparql = SPARQLWrapper(\"https://query.wikidata.org/sparql\", agent=agent)\n",
    "\n",
    "classes = {\n",
    "    \"Organization\": \"Q43229\", #includes companies\n",
    "    \"Name\": \"Q82799\",\n",
    "    \"Artist\": \"Q483501\",\n",
    "    \"Geolocation\": \"Q2221906\",\n",
    "    \"City\": \"Q515\",\n",
    "    \"Capital\": \"Q5119\",\n",
    "    \"Town\": \"Q3957\",\n",
    "    \"Demonym\": \"Q217438\",\n",
    "    \"Product\": \"Q2424752\",\n",
    "    \"Brand\": \"Q431289\",\n",
    "    \"Georegion\": \"Q82794\",\n",
    "    \"Country\": \"Q6256\",\n",
    "    \"Given name\": \"Q202444\",\n",
    "    \"Family name\": \"Q101352\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql'\n",
    "subclasses = {}\n",
    "\n",
    "for label, wikidata_code in tqdm(classes.items()):\n",
    "    try:\n",
    "        print(label, wikidata_code)\n",
    "        query = \"SELECT distinct ?class ?classLabel \" + \\\n",
    "                \"WHERE { ?class wdt:P279|wdt:P279/wdt:P279 wd:\" + wikidata_code + \". \" + \\\n",
    "                \"        SERVICE wikibase:label { bd:serviceParam wikibase:language 'en' } }\"\n",
    "        data = requests.get(url, params={'query': query, 'format': 'json'}).json()\n",
    "        # time.sleep(1)\n",
    "        subclasses[label] = data\n",
    "    except Exception as e:\n",
    "        print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subclasses = pickle.load(open('subclasses.pickle', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subclasses.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql'\n",
    "\n",
    "for label, wikidata_code in tqdm(classes.items()):\n",
    "    try:\n",
    "        if label not in [\"Family name\", \"Given name\"]:\n",
    "            continue\n",
    "        print(label, wikidata_code)\n",
    "        query = \"SELECT distinct ?class ?classLabel \" + \\\n",
    "                \"WHERE { ?class wdt:P279|wdt:P279/wdt:P279 wd:\" + wikidata_code + \". \" + \\\n",
    "                \"        SERVICE wikibase:label { bd:serviceParam wikibase:language 'en' } }\"\n",
    "        data = requests.get(url, params={'query': query, 'format': 'json'}).json()\n",
    "        # time.sleep(1)\n",
    "        subclasses[label] = data\n",
    "    except Exception as e:\n",
    "        print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# subclasses = pickle.load(open('subclasses.pickle', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for label in subclasses:\n",
    "    print(label)\n",
    "    print(len(subclasses[label]['results']['bindings']), 'subclasses')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(subclasses['Organization']['results']['bindings'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subclasses_data = []\n",
    "counter = 0\n",
    "for class_label in subclasses:\n",
    "    for result in subclasses[class_label]['results']['bindings']:\n",
    "        subclass, subclass_label = result['class']['value'], result['classLabel']['value']\n",
    "        if subclass_label.split('Q')[-1].isnumeric():\n",
    "            counter += 1\n",
    "            continue\n",
    "        \n",
    "        subclasses_data.append({ # 'class_uri': subclass,\n",
    "                                'class_qid': subclass.split('/')[-1],\n",
    "                                'class_label': class_label,\n",
    "                                'subclass_label': subclass_label.lower()})\n",
    "\n",
    "counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(subclasses, open('subclasses.pickle', 'wb'))\n",
    "subclasses_df = pd.DataFrame(subclasses_data)\n",
    "subclasses_df.to_csv('csv/subclasses_df.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subclasses_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(unused_classes).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(unused_classes).to_csv('csv/unused_subclasses.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Count the number of instances from every class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql'\n",
    "instances = {}\n",
    "\n",
    "for label, wikidata_code in classes.items():\n",
    "    try:\n",
    "        print(label, wikidata_code)\n",
    "        query = \"SELECT ?entity \" + \\\n",
    "                \"WHERE { ?entity wdt:P31/wdt:P279* wd:\" + wikidata_code + \". \" + \\\n",
    "                \"        SERVICE wikibase:label { bd:serviceParam wikibase:language 'en' } }\"\n",
    "        data = requests.get(url, params={'query': query, 'format': 'json'}).json()\n",
    "        instances[label] = data['results']['bindings']\n",
    "        # print(len(results[label]), 'of results returned.')\n",
    "    except Exception as e:\n",
    "        print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for label in classes:\n",
    "    print(label)\n",
    "    print(len(instances[label]), 'results')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the classes for each entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_classes.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/queries/examples#Cats\n",
    "# results_classes = {}\n",
    "limit = 10000\n",
    "\n",
    "for label, wikidata_code in tqdm(classes.items()):\n",
    "    if label != 'Family name':\n",
    "        continuesubclass2class = {}\n",
    "for key in subclasses:\n",
    "    for sc in subclasses[key]:\n",
    "        subclass2class[sc] = key\n",
    "pickle.dump(subclass2class, open('subclass2class.pickle', 'wb'))subclass2class = {}\n",
    "for key in subclasses:\n",
    "    for sc in subclasses[key]:\n",
    "        subclass2class[sc] = key\n",
    "pickle.dump(subclass2class, open('subclass2class.pickle', 'wb'))\n",
    "    print(label, wikidata_code)\n",
    "    results_classes[label] = []\n",
    "    offset = 0\n",
    "    while True:\n",
    "        try:\n",
    "             #  ?entity ?entityLabel ?entityAltLabel ?entityDescription\n",
    "            query = \"\"\"\n",
    "                    SELECT distinct  ?entity (GROUP_CONCAT(?class ; SEPARATOR = ';') as ?classes)\n",
    "                    WHERE \n",
    "                    {{\n",
    "                         ?entity wdt:P31|wdt:P31/wdt:P279|wdt:P31/wdt:P279/wdt:P279 wd:{};\n",
    "                                 wdt:P31 ?class.\n",
    "                    }}\n",
    "                    GROUP BY ?entity\n",
    "                    OFFSET {}\n",
    "                    LIMIT {}\n",
    "                    \"\"\".format(wikidata_code, offset*limit, limit)\n",
    "            if offset == 0:\n",
    "                print(query)\n",
    "            sparql.setQuery(query)\n",
    "            sparql.setReturnFormat(JSON)\n",
    "            offset += 1\n",
    "            if len(sparql.query().convert()['results']['bindings']) > 0:\n",
    "                result = sparql.query().convert()['results']['bindings']\n",
    "                results_classes[label].append(result)\n",
    "                print(f'{len(results_classes[label][-1])} of results returned at offset {offset}')\n",
    "            else:\n",
    "                print(f'No more results returned (offset {offset})')\n",
    "                break\n",
    "        except Exception as e:\n",
    "            if str(e).startswith('EndPointInternalError'):\n",
    "                e = 'Wikidata TimeoutException'\n",
    "                offset -= 1\n",
    "                time.sleep(30)\n",
    "            print(f'Exception at label {label} (offset {offset}) : {e}')\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(results_classes['Name'][0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_classes_data = []\n",
    "\n",
    "for class_label in results_classes:\n",
    "    results = [x for l in results_classes[class_label] for x in l]\n",
    "    for result in results:\n",
    "        entity_classes_data.append({\n",
    "            'entity_qid': result['entity']['value'].split('/')[-1],\n",
    "            'entity_label': class_label,\n",
    "            'entity_classes': ';'.join([s.split('/')[-1] for s in result['classes']['value'].split(';')]),\n",
    "        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(results_classes, open('results_classes.pickle', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_entity_classes = pd.DataFrame(entity_classes_data)\n",
    "df_entity_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_classes.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_entity_classes.to_csv('csv/entity_classes.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the labels for each entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# From https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/queries/examples#Cats\n",
    "labels_results = {}\n",
    "limit = 20000\n",
    "\n",
    "for label, wikidata_code in tqdm(classes.items()):\n",
    "    print(label, wikidata_code)\n",
    "    labels_results[label] = []\n",
    "    offset = 0\n",
    "    while True:\n",
    "        try:\n",
    "             #  ?entity ?entityLabel ?entityAltLabel ?entityDescription\n",
    "            query = \"\"\"\n",
    "                    SELECT DISTINCT ?entity ?entityLabel ?entityAltLabel ?entityDescription\n",
    "                    WHERE\n",
    "                    {{\n",
    "                      {{\n",
    "                        SELECT DISTINCT ?entity WHERE {{\n",
    "                          ?entity wdt:P31|wdt:P31/wdt:P279|wdt:P31/wdt:P279/wdt:P279 wd:{};\n",
    "                                  wdt:P31 ?class\n",
    "                        }}\n",
    "                        OFFSET {}\n",
    "                        LIMIT {}\n",
    "                      }}\n",
    "                      SERVICE wikibase:label {{ bd:serviceParam wikibase:language 'en' }}\n",
    "                    }}\n",
    "                    \"\"\".format(wikidata_code, offset*limit, limit)\n",
    "            if offset == 0:\n",
    "                print(query)\n",
    "            sparql.setQuery(query)\n",
    "            sparql.setReturnFormat(JSON)\n",
    "            offset += 1\n",
    "            if len(sparql.query().convert()['results']['bindings']) > 0:\n",
    "                result = sparql.query().convert()['results']['bindings']\n",
    "                labels_results[label].append(result)\n",
    "                print(f'{len(labels_results[label][-1])} of results returned at offset {offset}')\n",
    "            else:\n",
    "                print(f'No more results returned (offset {offset})')\n",
    "                break\n",
    "        except Exception as e:\n",
    "            print(f'Exception at label {label} (offset {offset}) : {e}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "limit = 10000\n",
    "\n",
    "for label, wikidata_code in tqdm(classes.items()):\n",
    "    if label != 'Family name':\n",
    "        continue\n",
    "    print(label, wikidata_code)\n",
    "    labels_results[label] = []\n",
    "    offset = 0\n",
    "    while True:\n",
    "        try:\n",
    "             #  ?entity ?entityLabel ?entityAltLabel ?entityDescription\n",
    "            query = \"\"\"\n",
    "                    SELECT DISTINCT ?entity ?entityLabel ?entityAltLabel ?entityDescription\n",
    "                    WHERE\n",
    "                    {{\n",
    "                      {{\n",
    "                        SELECT DISTINCT ?entity WHERE {{\n",
    "                          ?entity wdt:P31|wdt:P31/wdt:P279|wdt:P31/wdt:P279/wdt:P279 wd:{};\n",
    "                                  wdt:P31 ?class\n",
    "                        }}\n",
    "                        OFFSET {}\n",
    "                        LIMIT {}\n",
    "                      }}\n",
    "                      SERVICE wikibase:label {{ bd:serviceParam wikibase:language 'en' }}\n",
    "                    }}\n",
    "                    \"\"\".format(wikidata_code, offset*limit, limit)\n",
    "            if offset == 0:\n",
    "                print(query)\n",
    "            sparql.setQuery(query)\n",
    "            sparql.setReturnFormat(JSON)\n",
    "            offset += 1\n",
    "            if len(sparql.query().convert()['results']['bindings']) > 0:\n",
    "                result = sparql.query().convert()['results']['bindings']\n",
    "                labels_results[label].append(result)\n",
    "                print(f'{len(labels_results[label][-1])} of results returned at offset {offset}')\n",
    "            else:\n",
    "                print(f'No more results returned (offset {offset})')\n",
    "                break\n",
    "        except Exception as e:\n",
    "            print(f'Exception at label {label} (offset {offset}) : {e}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_results.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for l in labels_results:\n",
    "    print(l, len(labels_results[l]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(labels_results, open('gazetteers_v1.2_family_names_added_results.pickle', 'wb'))\n",
    "# labels_results = pickle.load(open('gazetteers_v1.0_results.pickle', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_results['Name'][0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_labels_data = []\n",
    "\n",
    "for class_label in tqdm(labels_results):\n",
    "    results = [x for l in labels_results[class_label] for x in l]\n",
    "    for result in results:\n",
    "        entity_labels_data.append({\n",
    "            'entity_qid': result['entity']['value'].split('/')[-1],\n",
    "            'entity_class': class_label,\n",
    "            'entity_label': '' if 'entityLabel' not in result else result['entityLabel']['value'],\n",
    "            'entity_alt_label': '' if 'entityAltLabel' not in result else result['entityAltLabel']['value'],\n",
    "            'entity_description':  '' if 'entityDescription' not in result else result['entityDescription']['value'],\n",
    "        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_entity_labels = pd.DataFrame(entity_labels_data)\n",
    "df_entity_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_entity_labels.to_csv('csv/entity_labels.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine Everything Together "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subclasses_df = pd.read_csv('csv/subclasses_df.csv')\n",
    "df_entity_labels = pd.read_csv('csv/entity_labels.csv')\n",
    "df_entity_classes = pd.read_csv('csv/entity_classes.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"alert alert-warning\">\n",
    "df_entity_labels contains more entries because the for Geolocation and Georegion it only goes 1 link deeper\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df_entity_labels))\n",
    "df_entity_labels.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df_entity_classes))\n",
    "df_entity_classes.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(subclasses_df))\n",
    "subclasses_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subclass2class = dict(zip(subclasses_df.subclass_label, subclasses_df.class_label.str.lower()))\n",
    "pickle.dump(subclass2class, open('subclass2class.pickle', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "qid2class = dict(zip(df_entity_labels.entity_qid, df_entity_labels.entity_class))\n",
    "qid2label = dict(zip(df_entity_labels.entity_qid, df_entity_labels.entity_label))\n",
    "qid2altlabel = dict(zip(df_entity_labels.entity_qid, df_entity_labels.entity_alt_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subclass2class = dict(zip(subclasses_df.class_qid, subclasses_df.class_label))\n",
    "subclass2label = dict(zip(subclasses_df.class_qid, subclasses_df.subclass_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_per_class = {}\n",
    "not_found = []\n",
    "\n",
    "for i, entry in tqdm(df_entity_classes.iterrows(), total=len(df_entity_classes)):\n",
    "    classes = entry['entity_classes']\n",
    "    label = entry['entity_label']\n",
    "    qid = entry['entity_qid']\n",
    "    \n",
    "    for cls in classes.split(';'):\n",
    "        if cls not in subclass2label:\n",
    "            not_found.append(cls)\n",
    "            cls_label = label.lower()\n",
    "        else:\n",
    "            cls_label = subclass2label[cls].lower()\n",
    "        \n",
    "        if cls_label not in entity_per_class:\n",
    "            entity_per_class[cls_label] = set()\n",
    "        \n",
    "        entity_per_class[cls_label].add(qid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "any(subclasses_df.subclass_label == 'male given name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "[l for l in entity_per_class.keys() if 'name' in l.lower()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subclass2labels = {}\n",
    "\n",
    "for subclass in entity_per_class:\n",
    "    subclass2labels[subclass] = set()\n",
    "    for entity in entity_per_class[subclass]:\n",
    "        if entity in qid2label and len(qid2label[entity].split(' ')) == 1:\n",
    "            subclass2labels[subclass].add(qid2label[entity].lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for l in subclass2labels:\n",
    "    if 'abdul' in subclass2labels[l]:\n",
    "        print(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "[(len(es), l) for l, es in sorted(subclass2labels.items(), key=lambda x: -len(x[1]))[:30]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label2subclasses = {}\n",
    "\n",
    "for subclass in subclass2labels:\n",
    "    for label in subclass2labels[subclass]:\n",
    "        if label not in label2subclasses:\n",
    "            label2subclasses[label] = []\n",
    "        label2subclasses[label].append(subclass)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label2subclasses['morocco']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(subclass2labels, open('subclass2labels.pickle', 'wb'))\n",
    "pickle.dump(label2subclasses, open('label2subclasses.pickle', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pick the final categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "[(len(es), l) for l, es in sorted(subclass2labels.items(), key=lambda x: -len(x[1]))[:300]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}