Untitled1 (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "63413b76-1929-4b0a-a446-37bfe35812fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from generate_graphs import *\n",
    "import tqdm\n",
    "import pandas as pd\n",
    "from generate_graphs_transformers import json_load_one2seq\n",
    "from utilities.utils import *\n",
    "from generate_graphs_transformers import *\n",
    "from utilities.utils import load_t5_preds, load_bart_preds\n",
    "\n",
    "def process_predictions(predictions, beam=False):\n",
    "    stemmer = PorterStemmer()\n",
    "\n",
    "    processed_predictions = []\n",
    "    for beam_prediction in predictions:\n",
    "        if beam:\n",
    "            prediction_ = \"\"\n",
    "            for prediction in beam_prediction:\n",
    "                prediction = prediction.replace(\";\", \"<sep>\")\n",
    "                prediction = prediction.split(\"<eos>\")[0]\n",
    "                if not prediction_:\n",
    "                    prediction_ += prediction\n",
    "                else:\n",
    "                    prediction_ += ' <sep> ' + prediction\n",
    "            prediction = prediction_\n",
    "        else:\n",
    "            beam_prediction = beam_prediction.replace(\";\", \"<sep>\")\n",
    "            prediction = beam_prediction.split(\"<eos>\")[0]\n",
    "\n",
    "        prediction = prediction.split(\",\")\n",
    "\n",
    "        stemed_prediction = []\n",
    "        for kp in prediction:\n",
    "            kp = kp.lower().strip()\n",
    "            if kp != \"\" and kp != \"<peos>\" and kp!=\",\" and kp != \".\" and kp != \"<unk>\":  # and \".\" not in kp and \",\" not in kp\n",
    "                tokenized_kp = kp.split(\" \")  # nltk.word_tokenize(kp)\n",
    "                tokenized_stemed_kp = [stemmer.stem(kw).strip() for kw in tokenized_kp]\n",
    "                stemed_kp = \" \".join(tokenized_stemed_kp).replace(\"< digit >\", \"<digit>\")\n",
    "                if stemed_kp.strip() != \"\":\n",
    "                    stemed_prediction.append(stemed_kp.strip())\n",
    "\n",
    "        # make prediction duplicates free but preserve order for @topk\n",
    "\n",
    "        prediction_dict = {}\n",
    "        stemed_prediction_ = []\n",
    "        for kp in stemed_prediction:\n",
    "            if kp not in prediction_dict:\n",
    "                prediction_dict[kp] = 1\n",
    "                stemed_prediction_.append(kp)\n",
    "        stemed_prediction = stemed_prediction_\n",
    "\n",
    "        processed_predictions.extend(stemed_prediction)\n",
    "\n",
    "    return processed_predictions\n",
    "\n",
    "def process_srcs(srcs):\n",
    "    stemmer = PorterStemmer()\n",
    "    processed_srcs = []\n",
    "    tokenized_src = srcs.split()  # Split the string into words\n",
    "    tokenized_stemed_src = [stemmer.stem(token.strip().lower()).strip() for token in tokenized_src]\n",
    "    stemed_src = \" \".join(tokenized_stemed_src).strip()\n",
    "    processed_srcs.append(stemed_src)\n",
    "    return processed_srcs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "34a21dd7-1f3d-4f61-a0eb-0edced286f6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_exhird_h(model,dataset):\n",
    "    scores, predictions, _ = json_load(model, dataset.lower())\n",
    "    return predictions \n",
    "\n",
    "def process_llama(dataset):\n",
    "    kp_predictions, probabilities, token_predictions, src, targets, all_kpp_values = load_llama(dataset)\n",
    "    token_predictions, scores = add_sep_and_kpp(token_predictions, probabilities)\n",
    "    predictions = [process_predictions(tp) for tp in kp_predictions]\n",
    "    context_lines = [process_srcs(ts) for ts in src]\n",
    "    gt_targets = [process_predictions(tt) for tt in targets]\n",
    "    return predictions, gt_targets, context_lines, kp_predictions\n",
    "\n",
    "def process_phi(dataset):\n",
    "    kp_predictions, probabilities, token_predictions, src, targets, all_kpp_values = load_phi(dataset)\n",
    "    token_predictions, scores = add_sep_and_kpp(token_predictions, probabilities)\n",
    "    predictions = [process_predictions(tp) for tp in kp_predictions]\n",
    "    context_lines = [process_srcs(ts) for ts in src]\n",
    "    gt_targets = [process_predictions(tt) for tt in targets]\n",
    "    return predictions, gt_targets, context_lines, kp_predictions\n",
    "\n",
    "def process_one2set(model,dataset):\n",
    "    scores, predictions, context_lines = json_load_one2set(model, dataset.lower())\n",
    "    return scores, predictions, context_lines\n",
    "\n",
    "def process_one2seq(model,dataset):\n",
    "    scores, predictions,entropies,context_lines = json_load_one2seq(model, dataset.lower())\n",
    "    return scores, predictions, context_lines\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "06b5f6c7-fefb-4842-a523-a0550b246f82",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_exhird_context(dataset):\n",
    "    path = 'data/test_datasets/processed_' + dataset + '_testing_context.txt'\n",
    "    target_file = open(path, encoding='utf-8')\n",
    "    target_lines = target_file.readlines()\n",
    "    targets = []\n",
    "    for i, preds in enumerate(target_lines):\n",
    "        preds = preds.split(';')\n",
    "        for j, pred in enumerate(preds):\n",
    "            preds[j] = pred.strip()\n",
    "        targets.append(preds)\n",
    "    return targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "51fdeda5-b51b-4241-aac8-7d1b860655f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def join_phrases_with_index(preds):\n",
    "    joined_phrases_with_index = []\n",
    "    for index, phrase_list in enumerate(preds):\n",
    "        temp_phrase = []\n",
    "        result = []\n",
    "        for word in phrase_list:\n",
    "            if word == '<sep>':\n",
    "                if temp_phrase:\n",
    "                    result.append(' '.join(temp_phrase))\n",
    "                    temp_phrase = []\n",
    "            else:\n",
    "                temp_phrase.append(word)\n",
    "        if temp_phrase:\n",
    "            result.append(' '.join(temp_phrase))\n",
    "        \n",
    "        joined_phrases_with_index.append(result)\n",
    "    return joined_phrases_with_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2142f72c-b025-44d0-9287-26f1e1ee1d05",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = 'one2seq_'  # Example model\n",
    "dataset = 'kp20k'  # Example dataset\n",
    "#context_lines=get_exhird_context(dataset)\n",
    "#preds = process_exhird_h(model, dataset)\n",
    "#src, pred, probabilities, predicted_tokens, targets = load_bart_preds(dataset)\n",
    "scores, pred, src = process_one2seq(model, dataset.lower())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1aca83f4-796d-4323-9f71-38c58ae1c88d",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "preds=[process_predictions(tp) for tp in join_phrases_with_index(pred)]\n",
    "\n",
    "\n",
    "context_lines = [process_srcs(ts) for ts in src]\n",
    "\n",
    "predictions = [process_predictions(tp) for tp in join_phrases_with_index(pred)]\n",
    "predictions=[list(set(target)) for target in predictions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7191d877-793a-4523-b0ae-aa94522c2401",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['feedback vertex set', 'degener graph', 'algorithm', 'graph algorithm'],\n",
       " ['data prefetch',\n",
       "  'perform',\n",
       "  'superscalar',\n",
       "  'microprocessor',\n",
       "  'memori latenc',\n",
       "  'design',\n",
       "  'experiment',\n",
       "  'measur',\n",
       "  'measur',\n",
       "  'cach hit model',\n",
       "  'cach perform']]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fc96d8ba-b0e4-4a15-b60b-43bf5d73b807",
   "metadata": {},
   "outputs": [],
   "source": [
    "#only for exhird\n",
    "#present_keywords_by_index = []\n",
    "#absent_keywords_by_index = []\n",
    "#\n",
    "## Iterate through preds\n",
    "#for pred in preds:\n",
    "#    present_keywords = []  # Present keyphrases for the current index\n",
    "#    absent_keywords = []   # Absent keyphrases for the current index\n",
    "#    kp_type = None  # Track whether we are in 'present' or 'absent' state\n",
    "#    current_phrase = []  # To accumulate the words of the current keyphrase\n",
    "#    \n",
    "#    for token in pred:\n",
    "#        if token == '<p_start>':\n",
    "#            kp_type = 'present'  # Switch to collecting present keyphrases\n",
    "#            current_phrase = []  # Reset the current keyphrase accumulator\n",
    "#        elif token == '<a_start>':\n",
    "#            kp_type = 'absent'  # Switch to collecting absent keyphrases\n",
    "#            current_phrase = []  # Reset the current keyphrase accumulator\n",
    "#        elif token == ';':  # End of the current keyphrase\n",
    "#            if kp_type == 'present' and current_phrase:\n",
    "#                present_keywords.append(' '.join(current_phrase))  # Join words to form the keyphrase\n",
    "#            elif kp_type == 'absent' and current_phrase:\n",
    "#                absent_keywords.append(' '.join(current_phrase))\n",
    "#            current_phrase = []  # Reset the keyphrase accumulator for the next phrase\n",
    "#        else:\n",
    "#            current_phrase.append(token)  # Accumulate the words of the current keyphrase\n",
    "#    \n",
    "#    # Append the present and absent keyphrases for the current index\n",
    "#    present_keywords_by_index.append(present_keywords)\n",
    "#    absent_keywords_by_index.append(absent_keywords)\n",
    "\n",
    "#preds = []\n",
    "#for present, absent in zip(present_keywords_by_index, absent_keywords_by_index):\n",
    "#    combined = present + absent  \n",
    "#    preds.append(combined)\n",
    "#predictions=[list(set(target)) for target in preds]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5250a894-a9b6-4d8c-8268-c286b43a1a78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "115336\n",
      "Overall average length: 5.77\n",
      "Number of keyphrases longer than average: 10055\n"
     ]
    }
   ],
   "source": [
    "def get_overall_average_keyphrase_length(predictions):\n",
    "    total_keyphrases = sum(len(target) for target in (predictions))\n",
    "    print(total_keyphrases)\n",
    "    return total_keyphrases / len(predictions)\n",
    "\n",
    "overall_average_length = get_overall_average_keyphrase_length(predictions)\n",
    "\n",
    "lists_higher_than_average = 0\n",
    "\n",
    "for keyphrase in predictions:\n",
    "    if len(keyphrase) > overall_average_length:\n",
    "        lists_higher_than_average += 1\n",
    "        \n",
    "print(f\"Overall average length: {overall_average_length:.2f}\")\n",
    "print(f\"Number of keyphrases longer than average: {lists_higher_than_average}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f9af6e40-26b5-4027-b8b5-794363419ac3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_keyphrases(predictions, context):\n",
    "    present_count = 0\n",
    "    absent_count = 0\n",
    "    # Convert context to a single string\n",
    "    context_string = \" \".join(context).lower()\n",
    "    for phrase in predictions:\n",
    "        if phrase.lower() in context_string:\n",
    "            present_count += 1\n",
    "        else:\n",
    "            absent_count += 1\n",
    "    return {\n",
    "        \"present_count\": present_count,\n",
    "        \"absent_count\": absent_count\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "89bad7ef-755d-4d52-ad61-72296998fbdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92290\n",
      "4.6145\n",
      "23046\n",
      "1.1523\n"
     ]
    }
   ],
   "source": [
    "results = [count_keyphrases(predictions[i], context_lines[i]) for i in range(len(predictions))]\n",
    "averages = []\n",
    "for entry in results:\n",
    "    present_count = entry['present_count']\n",
    "    absent_count = entry['absent_count']\n",
    "    total_count = present_count + absent_count\n",
    "    if present_count + absent_count==0:\n",
    "        total_count=1\n",
    "    average_present = present_count / total_count\n",
    "    average_absent = absent_count / total_count\n",
    "    averages.append((average_present, average_absent))\n",
    "\n",
    "# Calculate the overall average\n",
    "total_present_count = sum(entry['present_count'] for entry in results)\n",
    "total_absent_count = sum(entry['absent_count'] for entry in results)\n",
    "overall_average = total_absent_count / (total_present_count + total_absent_count)\n",
    "\n",
    "print(total_present_count)\n",
    "print(total_present_count/len(context_lines))\n",
    "print(total_absent_count)\n",
    "print(total_absent_count/len(context_lines))      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b0552f4a-9ee5-4377-b241-2a67abd24320",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45632"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(len(target) for target in (preds))-sum(len(target) for target in (predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3c0b1359-40ce-441b-b13e-c7606cd56b05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34888\n",
      "10744\n"
     ]
    }
   ],
   "source": [
    "results = [count_keyphrases(preds[i], context_lines[i]) for i in range(len(preds))]\n",
    "\n",
    "averages = []\n",
    "for entry in results:\n",
    "    present_count = entry['present_count']\n",
    "    absent_count = entry['absent_count']\n",
    "    total_count = present_count + absent_count\n",
    "    if present_count + absent_count==0:\n",
    "        total_count=1\n",
    "    average_present = present_count / total_count\n",
    "    average_absent = absent_count / total_count\n",
    "    averages.append((average_present, average_absent))\n",
    "\n",
    "# Calculate the overall average\n",
    "total_present_count_dup = sum(entry['present_count'] for entry in results)\n",
    "total_absent_count_dup = sum(entry['absent_count'] for entry in results)\n",
    "overall_average_dup = total_absent_count / (total_present_count + total_absent_count)\n",
    "\n",
    "print(total_present_count_dup-total_present_count)\n",
    "print(total_absent_count_dup-total_absent_count)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac7e80a2-53d6-4cdf-bdde-4f4ee34d94fa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fd39c3a-b940-4064-bfd5-381483c1085e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}