print_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import HTML, display, Markdown\n",
    "import itertools\n",
    "import tabulate\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import fileinput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_filename = 'input_data/seeds/seed_concepts'\n",
    "gold_filename = 'annotated-as-seed.json'\n",
    "# seed_filename = 'input_data/seeds/xlore_seeds.txt'\n",
    "topks = [100, 200, 500]\n",
    "relevance_field_names = ['', '', '', '']\n",
    "algorithms = ['tf_idf', 'pagerank', 'graph_prop', 'average_distance']\n",
    "raw_evaluated_filenames = [\n",
    "    f'processed_data/propagation_results/{a}_nf_result.json' for a in algorithms]\n",
    "rerank_evaluated_filenames = [\n",
    "    f'processed_data/rerank_results/{a}_nf_rerank_result.json' for a in algorithms]\n",
    "clustering_evaluated_filenames = [\n",
    "    f'processed_data/cluster_results/{a}_nf_cluster_result.json' for a in algorithms]\n",
    "clustering_evaluated_filenames = [\n",
    "    f'processed_data/cluster_results/{a}_nf_cluster_result.json' for a in algorithms]\n",
    "xlink_evaluated_filenames = ['processed_data/xlink_results/baike_context']\n",
    "from evaluation import load_json, evaluate, load_evaluated, merge_results\n",
    "with open(seed_filename) as f:\n",
    "    seeds = set([line.strip() for line in f])\n",
    "gold = set(load_json(gold_filename, lambda x: x['name']))\n",
    "gold = set(gold) - seeds\n",
    "results = {}\n",
    "\n",
    "def run_evaluate(evaluated_filenames):\n",
    "    results = {}\n",
    "    for (evaluated_filename, field_name), k in itertools.product(\n",
    "            zip(evaluated_filenames, relevance_field_names),\n",
    "            topks):\n",
    "        predicted = load_evaluated(evaluated_filename, gold, seeds, field_name)\n",
    "        result = evaluate(predicted, gold, k)\n",
    "        results[(evaluated_filename, k)] = result\n",
    "    results = merge_results(results)\n",
    "    return results\n",
    "    import collections\n",
    "    results = collections.OrderedDict(results)\n",
    "    \n",
    "    y_order = list(map(lambda x: ''.join(x), itertools.product(['mAP@', 'p@'], map(str, topks))))\n",
    "    results = {k: list(sorted(\n",
    "        v.items(),\n",
    "        key=lambda x: y_order.index(x[0]))) for k, v in results.items()}\n",
    "    return results\n",
    "\n",
    "def pandas_df_to_markdown_table(df):\n",
    "    from IPython.display import Markdown, display\n",
    "    fmt = ['---' for i in range(len(df.columns))]\n",
    "    df_fmt = pd.DataFrame([fmt], columns=df.columns)\n",
    "    df_formatted = pd.concat([df_fmt, df])\n",
    "    display(Markdown(df_formatted.to_csv(sep=\"|\", index=False)))\n",
    "\n",
    "def show_case(filename, gold, seeds, n_columns=8, n_shown=32, n_start=0):\n",
    "    predicted = load_evaluated(filename, gold, seeds)\n",
    "    # predicted = load_evaluated('processed_data/xlore_results/xlore_expansion.json', gold, seeds, '')\n",
    "    # predicted = load_evaluated('/data1/wyq/concept-expansion-snippet/tmp/baike_average_distance_nf_result.json', gold, seeds, '')\n",
    "    # pdp = list(map(lambda x: f'<font color=blue><h2>**{x}**</h2></font>' if x in gold else f'<h2>{x}</h2>', predicted[:500]))\n",
    "    pdp = list(map(lambda x: f'<font color=blue>**{x}**</font>' if x in gold else x, predicted[n_start:n_start+n_shown]))\n",
    "    def pack(l):\n",
    "        row = []\n",
    "        for i, p in enumerate(l):\n",
    "            i += 1 + n_start\n",
    "            row.append(p)\n",
    "            if i % n_columns == 0:\n",
    "                yield list(map(str, range(i-n_columns, i)))\n",
    "                yield row\n",
    "                row = []\n",
    "    pandas_df_to_markdown_table(pd.DataFrame(list(pack(pdp))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_idf</th>\n",
       "      <th>pagerank</th>\n",
       "      <th>graph_prop</th>\n",
       "      <th>average_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mAP@100</th>\n",
       "      <td>0.072228</td>\n",
       "      <td>0.012619</td>\n",
       "      <td>0.013567</td>\n",
       "      <td>0.274100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@100</th>\n",
       "      <td>0.210000</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.110000</td>\n",
       "      <td>0.470000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@200</th>\n",
       "      <td>0.045866</td>\n",
       "      <td>0.017023</td>\n",
       "      <td>0.016716</td>\n",
       "      <td>0.199214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@200</th>\n",
       "      <td>0.160000</td>\n",
       "      <td>0.145000</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.385000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@500</th>\n",
       "      <td>0.030162</td>\n",
       "      <td>0.015927</td>\n",
       "      <td>0.021165</td>\n",
       "      <td>0.110594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@500</th>\n",
       "      <td>0.142000</td>\n",
       "      <td>0.126000</td>\n",
       "      <td>0.152000</td>\n",
       "      <td>0.254000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           tf_idf  pagerank  graph_prop  average_distance\n",
       "mAP@100  0.072228  0.012619    0.013567          0.274100\n",
       "p@100    0.210000  0.140000    0.110000          0.470000\n",
       "mAP@200  0.045866  0.017023    0.016716          0.199214\n",
       "p@200    0.160000  0.145000    0.140000          0.385000\n",
       "mAP@500  0.030162  0.015927    0.021165          0.110594\n",
       "p@500    0.142000  0.126000    0.152000          0.254000"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(run_evaluate(raw_evaluated_filenames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tf_idf': {'mAP@100': 0.1072029399509533,\n",
       "  'p@100': 0.31,\n",
       "  'mAP@200': 0.07027217103150507,\n",
       "  'p@200': 0.22,\n",
       "  'mAP@500': 0.03762402350354539,\n",
       "  'p@500': 0.144},\n",
       " 'pagerank': {'mAP@100': 0.1329347913262355,\n",
       "  'p@100': 0.33,\n",
       "  'mAP@200': 0.07965611124507505,\n",
       "  'p@200': 0.22,\n",
       "  'mAP@500': 0.04131811533076112,\n",
       "  'p@500': 0.142},\n",
       " 'graph_prop': {'mAP@100': 0.286150182898988,\n",
       "  'p@100': 0.48,\n",
       "  'mAP@200': 0.1792996480197057,\n",
       "  'p@200': 0.33,\n",
       "  'mAP@500': 0.0838460314850487,\n",
       "  'p@500': 0.184},\n",
       " 'average_distance': {'mAP@100': 0.2207774012973763,\n",
       "  'p@100': 0.41,\n",
       "  'mAP@200': 0.15269420221010516,\n",
       "  'p@200': 0.325,\n",
       "  'mAP@500': 0.09077940817731159,\n",
       "  'p@500': 0.24}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_evaluate(rerank_evaluated_filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_idf_rerank</th>\n",
       "      <th>pagerank_rerank</th>\n",
       "      <th>graph_prop_rerank</th>\n",
       "      <th>average_distance_rerank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mAP@100</th>\n",
       "      <td>0.348924</td>\n",
       "      <td>0.314772</td>\n",
       "      <td>0.815018</td>\n",
       "      <td>0.628963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@100</th>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.530000</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>0.760000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@200</th>\n",
       "      <td>0.245946</td>\n",
       "      <td>0.232631</td>\n",
       "      <td>0.529913</td>\n",
       "      <td>0.435908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@200</th>\n",
       "      <td>0.435000</td>\n",
       "      <td>0.415000</td>\n",
       "      <td>0.595000</td>\n",
       "      <td>0.560000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@500</th>\n",
       "      <td>0.116060</td>\n",
       "      <td>0.113469</td>\n",
       "      <td>0.246153</td>\n",
       "      <td>0.229194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@500</th>\n",
       "      <td>0.232000</td>\n",
       "      <td>0.230000</td>\n",
       "      <td>0.320000</td>\n",
       "      <td>0.350000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tf_idf_rerank  pagerank_rerank  graph_prop_rerank  \\\n",
       "mAP@100       0.348924         0.314772           0.815018   \n",
       "p@100         0.580000         0.530000           0.850000   \n",
       "mAP@200       0.245946         0.232631           0.529913   \n",
       "p@200         0.435000         0.415000           0.595000   \n",
       "mAP@500       0.116060         0.113469           0.246153   \n",
       "p@500         0.232000         0.230000           0.320000   \n",
       "\n",
       "         average_distance_rerank  \n",
       "mAP@100                 0.628963  \n",
       "p@100                   0.760000  \n",
       "mAP@200                 0.435908  \n",
       "p@200                   0.560000  \n",
       "mAP@500                 0.229194  \n",
       "p@500                   0.350000  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict({k+'_rerank': v for k, v in run_evaluate(rerank_evaluated_filenames).items()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_idf_clustering</th>\n",
       "      <th>pagerank_clustering</th>\n",
       "      <th>graph_prop_clustering</th>\n",
       "      <th>average_distance_clustering</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mAP@100</th>\n",
       "      <td>0.891325</td>\n",
       "      <td>0.185312</td>\n",
       "      <td>0.708277</td>\n",
       "      <td>0.755868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@100</th>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.770000</td>\n",
       "      <td>0.810000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@200</th>\n",
       "      <td>0.666861</td>\n",
       "      <td>0.154414</td>\n",
       "      <td>0.457731</td>\n",
       "      <td>0.531812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@200</th>\n",
       "      <td>0.715000</td>\n",
       "      <td>0.365000</td>\n",
       "      <td>0.535000</td>\n",
       "      <td>0.615000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@500</th>\n",
       "      <td>0.302392</td>\n",
       "      <td>0.121055</td>\n",
       "      <td>0.244736</td>\n",
       "      <td>0.295524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@500</th>\n",
       "      <td>0.360000</td>\n",
       "      <td>0.318000</td>\n",
       "      <td>0.350000</td>\n",
       "      <td>0.418000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tf_idf_clustering  pagerank_clustering  graph_prop_clustering  \\\n",
       "mAP@100           0.891325             0.185312               0.708277   \n",
       "p@100             0.900000             0.420000               0.770000   \n",
       "mAP@200           0.666861             0.154414               0.457731   \n",
       "p@200             0.715000             0.365000               0.535000   \n",
       "mAP@500           0.302392             0.121055               0.244736   \n",
       "p@500             0.360000             0.318000               0.350000   \n",
       "\n",
       "         average_distance_clustering  \n",
       "mAP@100                     0.755868  \n",
       "p@100                       0.810000  \n",
       "mAP@200                     0.531812  \n",
       "p@200                       0.615000  \n",
       "mAP@500                     0.295524  \n",
       "p@500                       0.418000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict({k+'_clustering': v for k, v in run_evaluate(clustering_evaluated_filenames).items()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dsa_video_context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mAP@100</th>\n",
       "      <td>0.258177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@200</th>\n",
       "      <td>0.157677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@500</th>\n",
       "      <td>0.077051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@100</th>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@200</th>\n",
       "      <td>0.285000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@500</th>\n",
       "      <td>0.176000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         dsa_video_context\n",
       "mAP@100           0.258177\n",
       "mAP@200           0.157677\n",
       "mAP@500           0.077051\n",
       "p@100             0.400000\n",
       "p@200             0.285000\n",
       "p@500             0.176000"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(run_evaluate(['processed_data/xlink_results/dsa_video_context']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_idf</th>\n",
       "      <th>pagerank</th>\n",
       "      <th>graph_prop</th>\n",
       "      <th>average_distance</th>\n",
       "      <th>baike_context</th>\n",
       "      <th>baike_graph_prop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mAP@100</th>\n",
       "      <td>0.891325</td>\n",
       "      <td>0.185312</td>\n",
       "      <td>0.708277</td>\n",
       "      <td>0.755868</td>\n",
       "      <td>0.148936</td>\n",
       "      <td>0.013567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@100</th>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.770000</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>0.340000</td>\n",
       "      <td>0.110000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@200</th>\n",
       "      <td>0.666861</td>\n",
       "      <td>0.154414</td>\n",
       "      <td>0.457731</td>\n",
       "      <td>0.531812</td>\n",
       "      <td>0.112710</td>\n",
       "      <td>0.014875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@200</th>\n",
       "      <td>0.715000</td>\n",
       "      <td>0.365000</td>\n",
       "      <td>0.535000</td>\n",
       "      <td>0.615000</td>\n",
       "      <td>0.285000</td>\n",
       "      <td>0.130000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@500</th>\n",
       "      <td>0.302392</td>\n",
       "      <td>0.121055</td>\n",
       "      <td>0.244736</td>\n",
       "      <td>0.295524</td>\n",
       "      <td>0.068436</td>\n",
       "      <td>0.019391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@500</th>\n",
       "      <td>0.360000</td>\n",
       "      <td>0.318000</td>\n",
       "      <td>0.350000</td>\n",
       "      <td>0.418000</td>\n",
       "      <td>0.210000</td>\n",
       "      <td>0.146000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           tf_idf  pagerank  graph_prop  average_distance  baike_context  \\\n",
       "mAP@100  0.891325  0.185312    0.708277          0.755868       0.148936   \n",
       "p@100    0.900000  0.420000    0.770000          0.810000       0.340000   \n",
       "mAP@200  0.666861  0.154414    0.457731          0.531812       0.112710   \n",
       "p@200    0.715000  0.365000    0.535000          0.615000       0.285000   \n",
       "mAP@500  0.302392  0.121055    0.244736          0.295524       0.068436   \n",
       "p@500    0.360000  0.318000    0.350000          0.418000       0.210000   \n",
       "\n",
       "         baike_graph_prop  \n",
       "mAP@100          0.013567  \n",
       "p@100            0.110000  \n",
       "mAP@200          0.014875  \n",
       "p@200            0.130000  \n",
       "mAP@500          0.019391  \n",
       "p@500            0.146000  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(run_evaluate([\n",
    "'/data1/wyq/concept-expansion-snippet/tmp/baike_graph_prop_nf_result.json']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>xlore_expansion.json</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mAP@100</th>\n",
       "      <td>0.182173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@200</th>\n",
       "      <td>0.102730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@500</th>\n",
       "      <td>0.045215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@100</th>\n",
       "      <td>0.290000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@200</th>\n",
       "      <td>0.195000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@500</th>\n",
       "      <td>0.108000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         xlore_expansion.json\n",
       "mAP@100              0.182173\n",
       "mAP@200              0.102730\n",
       "mAP@500              0.045215\n",
       "p@100                0.290000\n",
       "p@200                0.195000\n",
       "p@500                0.108000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(run_evaluate(['processed_data/xlore_results/xlore_expansion.json']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.from_dict(run_evaluate([\n",
    "'/data1/wyq/concept-expansion-snippet/tmp/baike_average_distance_nf_result.json']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "0|1|2|3|4|5|6|7\n",
       "---|---|---|---|---|---|---|---\n",
       "0|1|2|3|4|5|6|7\n",
       "<font color=blue>**异或链表**</font>|<font color=blue>**基数排序**</font>|<font color=blue>**树状数组**</font>|<font color=blue>**循环链表**</font>|<font color=blue>**拓扑排序**</font>|拓朴排序|矩阵列运算|<font color=blue>**单向链表**</font>\n",
       "8|9|10|11|12|13|14|15\n",
       "<font color=blue>**马尔可夫算法**</font>|矩阵运算|<font color=blue>**算法和数据结构**</font>|<font color=blue>**递归**</font>|<font color=blue>**最优二叉树**</font>|<font color=blue>**关联数组**</font>|图,数据结构|数据结构列表\n",
       "16|17|18|19|20|21|22|23\n",
       "可定义函数|图论傅立叶转换|亏格|<font color=blue>**哈夫曼编码**</font>|控制流|<font color=blue>**数组步长**</font>|<font color=blue>**快速排序算法**</font>|<font color=blue>**十字链表**</font>\n",
       "24|25|26|27|28|29|30|31\n",
       "递归类型|<font color=blue>**位段**</font>|逻辑算子|<font color=blue>**递回**</font>|<font color=blue>**递归可枚举**</font>|<font color=blue>**逻辑运算符**</font>|<font color=blue>**图算法**</font>|二叉索引树\n",
       "32|33|34|35|36|37|38|39\n",
       "<font color=blue>**数据类型**</font>|分段函数|<font color=blue>**后缀数组**</font>|<font color=blue>**函数**</font>|集合,数据结构|双端伫列|平行数组|可计算函数\n",
       "40|41|42|43|44|45|46|47\n",
       "符号表|实数连续统|布尔运算符|数学函数|索引节点|路径,图论|哈密顿路径问题|树,数据结构\n",
       "48|49|50|51|52|53|54|55\n",
       "控制结构|结构归纳法|递归论|引理|哈希|欧拉路径|双端队列|多重关连数组\n",
       "56|57|58|59|60|61|62|63\n",
       "<font color=blue>**迭代**</font>|<font color=blue>**优先级队列**</font>|演算法|霍夫曼算法|字串演算法|冒泡算法|<font color=blue>**计算机算法**</font>|序列化\n",
       "64|65|66|67|68|69|70|71\n",
       "堆,数据结构|<font color=blue>**状态机**</font>|逻辑联结词|实数域|容器,数据结构|匹配,图论|概率图模型|平面图,图论\n",
       "72|73|74|75|76|77|78|79\n",
       "哈密尔顿路径问题|超平面|子集合|<font color=blue>**循环队列**</font>|函数,数学|图的遍历|稀疏矩阵|<font color=blue>**矩阵**</font>\n",
       "80|81|82|83|84|85|86|87\n",
       "图遍历|自动变量|容器,抽象数据类型|全函数|朱迪矩阵|汽泡排序法|等价关系|整数\n",
       "88|89|90|91|92|93|94|95\n",
       "树,图论|线段树|多元函数|随机生成树|基数查找树|约束变量|偏递归函数|偏函数\n"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "context_file = 'dsa_video_context_'\n",
    "algorithm = 'tf_idf_'\n",
    "cross_validation = 'cluster_'\n",
    "# seed_filename = 'input_data/seeds/xlore_seeds.txt'\n",
    "seed_filename = 'input_data/seeds/seed_concepts'\n",
    "with open(seed_filename) as f:\n",
    "    seeds = set([line.strip() for line in f])\n",
    "folder_dict = {'': 'propagation_results', 'cluster_': 'cluster_results', 'rerank_': 'rerank_results'}\n",
    "filename_seed_part = 'xlore_seed_' if seed_filename.endswith('xlore_seeds.txt') else 'more_seed_'\n",
    "context_file, algorithm, cross_validation, filename_seed_part = '', 'average_distance_', '', ''\n",
    "predicted = load_evaluated(\n",
    "    f'processed_data/{folder_dict[cross_validation]}/'\n",
    "    f'{context_file}{algorithm}{filename_seed_part}nf_{cross_validation}result.json', gold, seeds)\n",
    "predicted = load_evaluated('processed_data/xlore_results/xlore_expansion.json', gold, seeds, '')\n",
    "# predicted = load_evaluated('processed_data/xlink_results/baike_context', gold, seeds, '')\n",
    "# predicted = load_evaluated('/data1/wyq/concept-expansion-snippet/tmp/baike_average_distance_nf_result.json', gold, seeds, '')\n",
    "\n",
    "n_columns = 8\n",
    "# pdp = list(map(lambda x: f'<font color=blue><h2>**{x}**</h2></font>' if x in gold else f'<h2>{x}</h2>', predicted[:500]))\n",
    "pdp = list(map(lambda x: f'<font color=blue>**{x}**</font>' if x in gold else x, predicted[:100]))\n",
    "def pack(l):\n",
    "    row = []\n",
    "    for i, p in enumerate(l):\n",
    "        i += 1\n",
    "        row.append(p)\n",
    "        if i % n_columns == 0:\n",
    "            yield list(map(str, range(i-n_columns, i)))\n",
    "            yield row\n",
    "            row = []\n",
    "pdp = list(pack(pdp))\n",
    "# pdp = np.array(pdp).reshape(n_columns, -1)\n",
    "def pandas_df_to_markdown_table(df):\n",
    "    from IPython.display import Markdown, display\n",
    "    fmt = ['---' for i in range(len(df.columns))]\n",
    "    df_fmt = pd.DataFrame([fmt], columns=df.columns)\n",
    "    df_formatted = pd.concat([df_fmt, df])\n",
    "    display(Markdown(df_formatted.to_csv(sep=\"|\", index=False)))\n",
    "def pandas_df_to_html(df):\n",
    "    from IPython.display import HTML, display\n",
    "    df_fmt = pd.DataFrame([])\n",
    "\n",
    "# n_rows = 10a\n",
    "# n_columns = len(predicted) // n_rows + 1\n",
    "pandas_df_to_markdown_table(pd.DataFrame(pdp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['关键码', '字符', '向量', '复杂度', '运算符', '表达式', '指针', '概率', '红黑树', '操作数', '矩阵', '中位数', '数据元素', '时间复杂度', '最短路径', '左子', '变量', '控制权', '字符串', '无序', '叶子结点', '根结点', '右子', '操作符', '后缀', '内存', '对数', '单调性', '度量', '拓扑结构']\n"
     ]
    }
   ],
   "source": [
    "print(predicted[:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed_data/cluster_results/tf_idf_nf_cluster_result.json\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_idf</th>\n",
       "      <th>pagerank</th>\n",
       "      <th>average_distance</th>\n",
       "      <th>graph_prop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mAP@100</th>\n",
       "      <td>0.891325</td>\n",
       "      <td>0.262437</td>\n",
       "      <td>0.755868</td>\n",
       "      <td>0.708277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@100</th>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.490000</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>0.770000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@200</th>\n",
       "      <td>0.671268</td>\n",
       "      <td>0.202665</td>\n",
       "      <td>0.538480</td>\n",
       "      <td>0.457731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@200</th>\n",
       "      <td>0.720000</td>\n",
       "      <td>0.405000</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.535000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP@500</th>\n",
       "      <td>0.325005</td>\n",
       "      <td>0.147830</td>\n",
       "      <td>0.310766</td>\n",
       "      <td>0.258997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p@500</th>\n",
       "      <td>0.394000</td>\n",
       "      <td>0.340000</td>\n",
       "      <td>0.442000</td>\n",
       "      <td>0.374000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           tf_idf  pagerank  average_distance  graph_prop\n",
       "mAP@100  0.891325  0.262437          0.755868    0.708277\n",
       "p@100    0.900000  0.490000          0.810000    0.770000\n",
       "mAP@200  0.671268  0.202665          0.538480    0.457731\n",
       "p@200    0.720000  0.405000          0.625000    0.535000\n",
       "mAP@500  0.325005  0.147830          0.310766    0.258997\n",
       "p@500    0.394000  0.340000          0.442000    0.374000"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validation = 'cluster_'\n",
    "# seed_filename = 'input_data/seeds/xlore_seeds.txt'\n",
    "seed_filename = 'input_data/seeds/seed_concepts'\n",
    "folder_dict = {'': 'propagation_results', 'cluster_': 'cluster_results', 'rerank_': 'rerank_results'}\n",
    "filename_seed_part = 'xlore_seed_' if seed_filename.endswith('xlore_seeds.txt') else ''\n",
    "context_file = 'dsa_video_context_' if seed_filename.endswith('xlore_seeds.txt') else ''\n",
    "with open(seed_filename) as f:\n",
    "    seeds = set([line.strip() for line in f])\n",
    "gold = set(load_json(gold_filename, lambda x: x['name']))\n",
    "gold = set(gold) - seeds\n",
    "print(f'processed_data/{folder_dict[cross_validation]}/' \\\n",
    "    f'{context_file}tf_idf_{filename_seed_part}nf_{cross_validation}result.json')\n",
    "pd.DataFrame.from_dict(run_evaluate([\n",
    "    f'processed_data/{folder_dict[cross_validation]}/'\n",
    "    f'{context_file}{algorithm}{filename_seed_part}nf_{cross_validation}result.json' \\\n",
    "    for algorithm in ['tf_idf_', 'pagerank_', 'average_distance_', 'graph_prop_']]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed_data/cluster_results/tf_idf_nf_cluster_result.json\n",
      "{1: 0, 2: 467, 3: 883, 4: 1092, 5: 1513, 6: 1982, 7: 2127, 8: 2474, 9: 2815, 10: 3356, 11: 3637, 12: 4028, 13: 4235, 14: 4703, 15: 5109}\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "0|1|2|3|4|5|6|7\n",
       "---|---|---|---|---|---|---|---\n",
       "320|321|322|323|324|325|326|327\n",
       "<font color=blue>**汉诺塔**</font>|剖面图|图形|<font color=blue>**全排列**</font>|方法|分析|排除法|相似图形\n",
       "328|329|330|331|332|333|334|335\n",
       "独立同分布|分析机|法的实现|<font color=blue>**分组**</font>|定性分析|定界|第一性|可行域\n",
       "336|337|338|339|340|341|342|343\n",
       "甘特图|相邻关系|汇总|<font color=blue>**回溯法**</font>|图像恢复|优先|<font color=blue>**启发式**</font>|广义表\n",
       "344|345|346|347|348|349|350|351\n",
       "流程图|分析法|标准流程|耦合|参考模型|对称多处理|空间划分|资源分配\n",
       "352|353|354|355|356|357|358|359\n",
       "方案选择|构造器|分治策略|排队|<font color=blue>**参数化**</font>|基本结构|结构|<font color=blue>**流水线法**</font>\n",
       "360|361|362|363|364|365|366|367\n",
       "分析器|实例化|分支过程|<font color=blue>**先进先出**</font>|示意图|随机行走|任务并行|凸规划\n"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "algorithm = 'tf_idf_'\n",
    "filename = f'processed_data/{folder_dict[cross_validation]}/' \\\n",
    "    f'{context_file}{algorithm}{filename_seed_part}nf_{cross_validation}result.json'\n",
    "print(filename)\n",
    "def find_cluster_start(filename):\n",
    "    cluster_start = {}\n",
    "    with open(filename) as f:\n",
    "        for i, line in enumerate(f):\n",
    "            cluster_num = int(json.loads(line)['cluster'])\n",
    "            if cluster_num not in cluster_start:\n",
    "                cluster_start[cluster_num] = i\n",
    "    return cluster_start\n",
    "def show_all_cases(filename, first_n):\n",
    "    cluster_start = find_cluster_start(filename)\n",
    "    print(cluster_start)\n",
    "    my_gold = set(load_json(gold_filename, lambda x: x['name']))\n",
    "    show_case(filename, my_gold, {}, n_start=320, n_shown=48)\n",
    "#     for i in range(first_n):\n",
    "#         show_case(filename, my_gold, {}, n_start=cluster_start[i+1], n_shown=48)\n",
    "show_all_cases(filename, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{' 图论算法',\n",
       " 'KMP算法',\n",
       " 'VLSI并行算法',\n",
       " 'a*算法',\n",
       " 'avl树',\n",
       " 'b+树',\n",
       " 'bellman-ford算法',\n",
       " 'bsp模型',\n",
       " 'b树',\n",
       " 'c++',\n",
       " 'c函数',\n",
       " 'deque',\n",
       " 'dijkstra算法',\n",
       " 'distance',\n",
       " 'dna计算',\n",
       " 'd算法',\n",
       " 'floyd-warshall算法',\n",
       " 'floyd算法',\n",
       " 'hash',\n",
       " 'hash算法',\n",
       " 'k-means算法',\n",
       " 'kosaraju算法',\n",
       " 'kruskal算法',\n",
       " 'loop',\n",
       " 'null值',\n",
       " 'pad图',\n",
       " 'postfix',\n",
       " 'pram模型',\n",
       " 'prim算法',\n",
       " 'spfa算法',\n",
       " 'tarjan算法',\n",
       " 'ρ(n )近似算法',\n",
       " '一维数组',\n",
       " '下界',\n",
       " '不可近似性',\n",
       " '中位数',\n",
       " '中根遍历',\n",
       " '中缀',\n",
       " '中缀表达式',\n",
       " '串匹配',\n",
       " '乘法原理',\n",
       " '乱序',\n",
       " '二元关系',\n",
       " '二分图',\n",
       " '二分插入',\n",
       " '二分搜索',\n",
       " '二分查找',\n",
       " '二分法',\n",
       " '二叉堆',\n",
       " '二叉排序树',\n",
       " '二叉搜索树',\n",
       " '二叉查找树',\n",
       " '二叉树算法',\n",
       " '二叉树遍历',\n",
       " '二维数组',\n",
       " '二维网孔',\n",
       " '二路归并',\n",
       " '二进制数字',\n",
       " '二进制编码',\n",
       " '二部图',\n",
       " '交互计算',\n",
       " '代数系统',\n",
       " '众核计算',\n",
       " '优先级',\n",
       " '优先级队列',\n",
       " '伪代码',\n",
       " '伪多项式时间算法',\n",
       " '伪随机数',\n",
       " '似然函数',\n",
       " '位段',\n",
       " '例程',\n",
       " '信包选路问题',\n",
       " '信号流图',\n",
       " '倍增技术',\n",
       " '假溢出',\n",
       " '偏移量',\n",
       " '儿子',\n",
       " '元启发式优化算法',\n",
       " '元启发式算法',\n",
       " '元数据',\n",
       " '元组',\n",
       " '兄弟',\n",
       " '兄弟结点',\n",
       " '先序遍历',\n",
       " '先根遍历',\n",
       " '先进先出',\n",
       " '免疫遗传算法',\n",
       " '入队',\n",
       " '全排列',\n",
       " '关联数组',\n",
       " '关键字',\n",
       " '关键码',\n",
       " '关键词',\n",
       " '关键路径法',\n",
       " '内点',\n",
       " '内点算法',\n",
       " '内部排序',\n",
       " '冯诺依曼',\n",
       " '决策树',\n",
       " '函数',\n",
       " '函数式编程',\n",
       " '函数模型',\n",
       " '函数调用语句',\n",
       " '分布式排序算法',\n",
       " '分布式计算技术',\n",
       " '分支',\n",
       " '分支定界法',\n",
       " '分支限界',\n",
       " '分枝限界法',\n",
       " '分治算法',\n",
       " '分类器',\n",
       " '分组',\n",
       " '分配函数',\n",
       " '划分技术',\n",
       " '划分算法',\n",
       " '初始化',\n",
       " '前驱',\n",
       " '加权图',\n",
       " '加法原理',\n",
       " '动态分配内存',\n",
       " '动态数组',\n",
       " '动态规划法',\n",
       " '动态规划算法',\n",
       " '动态链表',\n",
       " '十六进制数',\n",
       " '十字链表',\n",
       " '单向函数',\n",
       " '单向散列函数',\n",
       " '单向连通图',\n",
       " '单向链表',\n",
       " '单形',\n",
       " '单源最短路径',\n",
       " '单纯形',\n",
       " '单纯形表',\n",
       " '单调队列',\n",
       " '原地',\n",
       " '原地排序',\n",
       " '原地算法',\n",
       " '参数',\n",
       " '参数估计',\n",
       " '参数化',\n",
       " '参数表',\n",
       " '叉',\n",
       " '双连通分量',\n",
       " '反向传播算法',\n",
       " '变长字符串',\n",
       " '变长数组',\n",
       " '可扩放性',\n",
       " '可行区域',\n",
       " '叶',\n",
       " '叶子',\n",
       " '叶子结点',\n",
       " '叶子节点',\n",
       " '叶结点',\n",
       " '合并同类项',\n",
       " '合数问题',\n",
       " '同步并行算法',\n",
       " '同步算法',\n",
       " '后序',\n",
       " '后根遍历',\n",
       " '后继',\n",
       " '后缀数组',\n",
       " '后缀表达式',\n",
       " '向量',\n",
       " '向量结构',\n",
       " '启发式',\n",
       " '启发式优化算法',\n",
       " '启发式方法',\n",
       " '启发式算法',\n",
       " '哈夫曼',\n",
       " '哈夫曼算法',\n",
       " '哈夫曼编码',\n",
       " '哈密顿回路问题',\n",
       " '哈希值',\n",
       " '回溯',\n",
       " '回溯法',\n",
       " '回溯算法',\n",
       " '回滚',\n",
       " '团',\n",
       " '图',\n",
       " '图同构',\n",
       " '图形结构',\n",
       " '图模型',\n",
       " '图灵机',\n",
       " '图着色问题',\n",
       " '图算法',\n",
       " '图论',\n",
       " '图论算法',\n",
       " '埃德蒙兹-卡普算法',\n",
       " '基数排序',\n",
       " '堆',\n",
       " '堆栈',\n",
       " '备忘录算法',\n",
       " '复杂性',\n",
       " '外路长度',\n",
       " '外部排序',\n",
       " '多核计算',\n",
       " '多维数组',\n",
       " '多项式时间算法',\n",
       " '多项式时间近似格式',\n",
       " '头指针',\n",
       " '头结点',\n",
       " '子程序',\n",
       " '字符',\n",
       " '字符串匹配',\n",
       " '字符串表',\n",
       " '字符类',\n",
       " '孩子',\n",
       " '完全多项式时间近似格式',\n",
       " '完全有向图',\n",
       " '宏流水线算法',\n",
       " '实时计算',\n",
       " '宽度优先搜索',\n",
       " '密码算法',\n",
       " '对象指针',\n",
       " '寻路',\n",
       " '尾递归',\n",
       " '尾部递归',\n",
       " '希尔排序',\n",
       " '希尔排序算法',\n",
       " '平均性态分析',\n",
       " '平方取中法',\n",
       " '平衡二叉树',\n",
       " '平衡树',\n",
       " '平衡树方法',\n",
       " '并发程序',\n",
       " '并发计算',\n",
       " '并行图论算法',\n",
       " '并行外排序',\n",
       " '并行排序算法',\n",
       " '并行算法的成本',\n",
       " '并行计算模型',\n",
       " '并行选择算法',\n",
       " '并行随机存取机',\n",
       " '广度优先搜索',\n",
       " '广度优先算法',\n",
       " '广度优先遍历',\n",
       " '序列',\n",
       " '序列比对',\n",
       " '度数',\n",
       " '开放寻址法',\n",
       " '异或链表',\n",
       " '异步并行算法',\n",
       " '异步算法',\n",
       " '弱连通图',\n",
       " '归并插入',\n",
       " '形函数',\n",
       " '循环',\n",
       " '循环体',\n",
       " '循环图',\n",
       " '循环数',\n",
       " '循环链表',\n",
       " '循环队列',\n",
       " '微粒群优化算法',\n",
       " '微粒群算法',\n",
       " '快速傅立叶变换',\n",
       " '快速排序',\n",
       " '快速排序算法',\n",
       " '折半插入排序',\n",
       " '拓扑排序',\n",
       " '拓扑结构',\n",
       " '指数级',\n",
       " '指派问题',\n",
       " '指针',\n",
       " '指针变量',\n",
       " '指针数组',\n",
       " '排序',\n",
       " '排序网络',\n",
       " '描述数据',\n",
       " '插入序列',\n",
       " '插入法',\n",
       " '搜索引擎算法',\n",
       " '搜索算法',\n",
       " '支数',\n",
       " '改进单纯形法',\n",
       " '散列',\n",
       " '散列函数',\n",
       " '散列技术',\n",
       " '散列法',\n",
       " '散列算法',\n",
       " '散列表',\n",
       " '数值计算',\n",
       " '数列',\n",
       " '数学建模',\n",
       " '数据',\n",
       " '数据排序',\n",
       " '数据模型',\n",
       " '数据流',\n",
       " '数据流图',\n",
       " '数据的相关关系',\n",
       " '数据类型',\n",
       " '数据组织',\n",
       " '数据结构',\n",
       " '数据缓冲区',\n",
       " '数组元素',\n",
       " '数组步长',\n",
       " '数量级',\n",
       " '断链',\n",
       " '旅行推销员问题',\n",
       " '无向完全图',\n",
       " '无序',\n",
       " '时间分析',\n",
       " '时间复杂度',\n",
       " '时间复杂性',\n",
       " '时间空间权衡',\n",
       " '普里姆算法',\n",
       " '最优二叉树',\n",
       " '最优决策',\n",
       " '最优化问题',\n",
       " '最优子结构性',\n",
       " '最优并行算法',\n",
       " '最优归并树',\n",
       " '最坏情况时间复杂性',\n",
       " '最大值问题',\n",
       " '最大团',\n",
       " '最小值问题',\n",
       " '最短路径快速算法',\n",
       " '最短路问题',\n",
       " '最长公共子序列',\n",
       " '有序',\n",
       " '有序数组',\n",
       " '有序树',\n",
       " '有效并行算法',\n",
       " '朴素贝叶斯',\n",
       " '权',\n",
       " '权值',\n",
       " '权重',\n",
       " '权重值',\n",
       " '析构函数',\n",
       " '枚举算法',\n",
       " '标准函数',\n",
       " '栈',\n",
       " '栈顶指针',\n",
       " '树压缩技术',\n",
       " '树型',\n",
       " '树形',\n",
       " '树形图',\n",
       " '树形选择排序',\n",
       " '树形链接',\n",
       " '树状数组',\n",
       " '树的遍历',\n",
       " '树类',\n",
       " '根结点',\n",
       " '检索树',\n",
       " '森林',\n",
       " '概率并行算法',\n",
       " '模拟退火',\n",
       " '模拟退火法',\n",
       " '次序',\n",
       " '正则',\n",
       " '比较器网络',\n",
       " '汉诺塔',\n",
       " '泛型',\n",
       " '波前算法',\n",
       " '波前阵列',\n",
       " '洗牌交换链接',\n",
       " '流',\n",
       " '流水线法',\n",
       " '消息队列',\n",
       " '深度优先遍历',\n",
       " '混合整数规划',\n",
       " '源',\n",
       " '源点',\n",
       " '演化计算',\n",
       " '爬山算法',\n",
       " '父结点',\n",
       " '物理地址',\n",
       " '特权',\n",
       " '状态机',\n",
       " '状态转移方程',\n",
       " '独立数',\n",
       " '独立集问题',\n",
       " '理论模型',\n",
       " '理论计算机科学',\n",
       " '用户自定义函数',\n",
       " '直接插入排序',\n",
       " '直接选择排序',\n",
       " '着色数目问题',\n",
       " '矢量数据',\n",
       " '矩阵',\n",
       " '破对称技术',\n",
       " '确定性模型',\n",
       " '确定选路算法',\n",
       " '祖先',\n",
       " '神经网络',\n",
       " '神经网络模型',\n",
       " '神经网络算法',\n",
       " '福特-富尔克森方法',\n",
       " '稀疏图',\n",
       " '稳定性分析',\n",
       " '稳定排序',\n",
       " '稳定排序算法',\n",
       " '穷举法',\n",
       " '空图',\n",
       " '空间复杂度',\n",
       " '等价类划分',\n",
       " '简单图',\n",
       " '简单排序',\n",
       " '简单路径',\n",
       " '算法与数据结构',\n",
       " '算法博弈论',\n",
       " '算法和数据结构',\n",
       " '算法基础',\n",
       " '算法复杂度',\n",
       " '算法导论',\n",
       " '算法效率',\n",
       " '算法正确性',\n",
       " '算法策略',\n",
       " '算法设计与分析',\n",
       " '箱排序',\n",
       " '米勒-罗宾算法',\n",
       " '类型转换',\n",
       " '粒子群优化',\n",
       " '粒子群算法',\n",
       " '精确算法',\n",
       " '系统树',\n",
       " '索引',\n",
       " '索引数组',\n",
       " '索林算法',\n",
       " '约束',\n",
       " '约翰逊算法',\n",
       " '纳什均衡',\n",
       " '线性',\n",
       " '线性关系',\n",
       " '线性化',\n",
       " '线性规划法',\n",
       " '线性链表',\n",
       " '线性阵列',\n",
       " '线索化',\n",
       " '线路交换模式',\n",
       " '组合优化',\n",
       " '组合最优化',\n",
       " '经典数据结构',\n",
       " '结构化数据',\n",
       " '结点',\n",
       " '统计模型',\n",
       " '编码',\n",
       " '编译程序',\n",
       " '网格计算',\n",
       " '网络分析',\n",
       " '网络拓扑结构',\n",
       " '罗宾-泽利科夫斯基算法',\n",
       " '群智能算法',\n",
       " '脉动算法',\n",
       " '自调度算法',\n",
       " '蒙特卡罗算法',\n",
       " '蚁群算法',\n",
       " '蝶形运算',\n",
       " '表达式',\n",
       " '表达式树',\n",
       " '覆盖问题',\n",
       " '规划',\n",
       " '规划法',\n",
       " '规划算法',\n",
       " '解线性规划',\n",
       " '计数器',\n",
       " '计数排序',\n",
       " '计算复杂性',\n",
       " '计算机',\n",
       " '计算机科学',\n",
       " '计算机程序',\n",
       " '计算机算法',\n",
       " '语句块',\n",
       " '调度问题',\n",
       " '谢尔排序',\n",
       " '贝切尔比较器',\n",
       " '贝叶斯分类器',\n",
       " '贪婪算法',\n",
       " '贪心算法',\n",
       " '贪心选路算法',\n",
       " '资源约束',\n",
       " '赋值函数',\n",
       " '赋值语句',\n",
       " '赋权图',\n",
       " '起泡排序',\n",
       " '超立方体链接',\n",
       " '越界',\n",
       " '路径',\n",
       " '路由算法',\n",
       " '车辆路径问题',\n",
       " '转义字符',\n",
       " '输入节点',\n",
       " '输入规模',\n",
       " '边表',\n",
       " '近似格式',\n",
       " '近似算法的性能比',\n",
       " '近似算法的相对误差',\n",
       " '进化算法',\n",
       " '连通分支',\n",
       " '连通度',\n",
       " '迪杰斯特拉算法',\n",
       " '迪科斯彻算法',\n",
       " '迭代',\n",
       " '迭代法',\n",
       " '迭代计算',\n",
       " '逆序',\n",
       " '逆序数',\n",
       " '逆波兰表达式',\n",
       " '选址问题',\n",
       " '选择网络',\n",
       " '递回',\n",
       " '递归',\n",
       " '递归函数',\n",
       " '递归可枚举',\n",
       " '递归定义',\n",
       " '递归法',\n",
       " '递归程序',\n",
       " '递归过程',\n",
       " '递推关系',\n",
       " '通路',\n",
       " '逻辑结构设计',\n",
       " '逻辑表达式',\n",
       " '逻辑运算符',\n",
       " '遍历',\n",
       " '遗传优化算法',\n",
       " '遗传算法',\n",
       " '量子计算',\n",
       " '链',\n",
       " '键',\n",
       " '键值',\n",
       " '键值对',\n",
       " '闭包',\n",
       " '闭合式',\n",
       " '队列',\n",
       " '随机决策',\n",
       " '随机函数',\n",
       " '随机化算法',\n",
       " '随机增量构造',\n",
       " '随机并行算法',\n",
       " '随机序列',\n",
       " '随机搜索',\n",
       " '随机数',\n",
       " '随机游走模型',\n",
       " '随机算法',\n",
       " '随机规划',\n",
       " '随机过程',\n",
       " '随机选路算法',\n",
       " '霍夫曼',\n",
       " '静态数据',\n",
       " '静态规划',\n",
       " '静态选路',\n",
       " '静态链表',\n",
       " '非对称算法',\n",
       " '非线性规划',\n",
       " '页表',\n",
       " '顶点',\n",
       " '顶点图',\n",
       " '顺序',\n",
       " '顺序存储结构',\n",
       " '顺序执行',\n",
       " '顺序结构',\n",
       " '预调度算法',\n",
       " '马尔可夫算法',\n",
       " '鲁棒性',\n",
       " '鸽巢排序'}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}