analysis-of-submissions/tira-ir-replicability.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c8888d0c-d17c-4d5f-8cc7-5d510e2eec7a",
   "metadata": {},
   "source": [
    "# Replicability Accross Different Corpora: A Case Study with Software Submissions in TIRA"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38eab5e1-b4a7-4ba8-8124-a1337e65a9f4",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Install Dependencies\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "12f29139-2038-4bf9-82ac-de8b8f720f21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://github.com/tira-io/tira.git@development#subdirectory=python-client\n",
      "  Cloning https://github.com/tira-io/tira.git (to revision development) to /tmp/pip-req-build-n8ef1e9_\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/tira-io/tira.git /tmp/pip-req-build-n8ef1e9_\n",
      "  Running command git checkout -b development --track origin/development\n",
      "  Switched to a new branch 'development'\n",
      "  Branch 'development' set up to track remote branch 'development' from 'origin'.\n",
      "  Resolved https://github.com/tira-io/tira.git to commit 4217559835ad59e9b91c6fc1f2ebca70d123c896\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: requests==2.*,>=2.26 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from tira==0.0.11) (2.28.1)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from requests==2.*,>=2.26->tira==0.0.11) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from requests==2.*,>=2.26->tira==0.0.11) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from requests==2.*,>=2.26->tira==0.0.11) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from requests==2.*,>=2.26->tira==0.0.11) (1.26.12)\n",
      "Requirement already satisfied: repro_eval==0.4.0 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (0.4.0)\n",
      "Requirement already satisfied: pytrec-eval in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from repro_eval==0.4.0) (0.5)\n",
      "Requirement already satisfied: GitPython in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from repro_eval==0.4.0) (3.1.30)\n",
      "Requirement already satisfied: py-cpuinfo in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from repro_eval==0.4.0) (9.0.0)\n",
      "Requirement already satisfied: numpy in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from repro_eval==0.4.0) (1.23.5)\n",
      "Requirement already satisfied: tqdm in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from repro_eval==0.4.0) (4.64.1)\n",
      "Requirement already satisfied: ruamel.yaml in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from repro_eval==0.4.0) (0.17.21)\n",
      "Requirement already satisfied: scipy in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from repro_eval==0.4.0) (1.9.3)\n",
      "Requirement already satisfied: gitdb<5,>=4.0.1 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from GitPython->repro_eval==0.4.0) (4.0.10)\n",
      "Requirement already satisfied: ruamel.yaml.clib>=0.2.6 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ruamel.yaml->repro_eval==0.4.0) (0.2.7)\n",
      "Requirement already satisfied: smmap<6,>=3.0.1 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from gitdb<5,>=4.0.1->GitPython->repro_eval==0.4.0) (5.0.0)\n",
      "Collecting git+https://github.com/allenai/ir_datasets.git\n",
      "  Cloning https://github.com/allenai/ir_datasets.git to /tmp/pip-req-build-uvp15174\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/allenai/ir_datasets.git /tmp/pip-req-build-uvp15174\n",
      "  Resolved https://github.com/allenai/ir_datasets.git to commit 280164ab597fb43bbb5f97f827ab5fa8e74f780e\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: beautifulsoup4>=4.4.1 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (4.11.1)\n",
      "Requirement already satisfied: inscriptis>=2.2.0 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (2.3.2)\n",
      "Requirement already satisfied: lxml>=4.5.2 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (4.9.1)\n",
      "Requirement already satisfied: numpy>=1.18.1 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (1.23.5)\n",
      "Requirement already satisfied: pyyaml>=5.3.1 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (6.0)\n",
      "Requirement already satisfied: requests>=2.22.0 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (2.28.1)\n",
      "Requirement already satisfied: tqdm>=4.38.0 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (4.64.1)\n",
      "Requirement already satisfied: trec-car-tools>=2.5.4 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (2.6)\n",
      "Requirement already satisfied: lz4>=3.1.10 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (4.0.2)\n",
      "Requirement already satisfied: warc3-wet>=0.2.3 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (0.2.3)\n",
      "Requirement already satisfied: warc3-wet-clueweb09>=0.2.5 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (0.2.5)\n",
      "Requirement already satisfied: zlib-state>=0.1.3 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (0.1.5)\n",
      "Requirement already satisfied: ijson>=3.1.3 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (3.1.4)\n",
      "Requirement already satisfied: pyautocorpus>=0.1.1 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (0.1.9)\n",
      "Requirement already satisfied: unlzw3>=0.2.1 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from ir-datasets==0.5.4) (0.2.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from beautifulsoup4>=4.4.1->ir-datasets==0.5.4) (2.3.2.post1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from requests>=2.22.0->ir-datasets==0.5.4) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from requests>=2.22.0->ir-datasets==0.5.4) (2022.12.7)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from requests>=2.22.0->ir-datasets==0.5.4) (2.1.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from requests>=2.22.0->ir-datasets==0.5.4) (1.26.12)\n",
      "Requirement already satisfied: cbor>=1.0.0 in /home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages (from trec-car-tools>=2.5.4->ir-datasets==0.5.4) (1.0.0)\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install git+https://github.com/tira-io/tira.git@development#subdirectory=python-client\n",
    "!{sys.executable} -m pip install repro_eval==0.4.0\n",
    "!{sys.executable} -m pip install git+https://github.com/allenai/ir_datasets.git"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a533c1a-0c72-4e54-addd-fae2d71b99d2",
   "metadata": {},
   "source": [
    "### Import modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "d53ee110-a9b3-4866-9192-0a2a874324c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tira.rest_api_client import Client\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import ir_datasets\n",
    "from repro_eval.Evaluator import RpdEvaluator, RplEvaluator\n",
    "from repro_eval.util import arp, arp_scores, print_base_adv, print_simple_line, trim\n",
    "\n",
    "from pathlib import Path\n",
    "import glob\n",
    "import json\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "740271f3-2750-4e96-af4a-a4c2ed78accf",
   "metadata": {},
   "source": [
    "### Instantiate the TIRA client and define the task\n",
    "This evaluation is based on the experiments from the TIRA task \"ir-benchmarks\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c48c144b-b336-4be5-98f8-9a5ca58c519e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get your API Key at: https://tira.io -> Login -> Admin -> API -> New API Key\n",
    "tira = Client(api_key='<API-KEY-OMMITTED>')\n",
    "\n",
    "TASK = 'ir-benchmarks'"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "2a08121b-fefc-4a9e-b508-e6276053513a",
   "metadata": {},
   "source": [
    "### Load all evaluations over all datasets of the given TIRA task\n",
    "If the file does not already exist locally, the evaluations over each dataset from the TIRA task are downloaded from TIRA and then stored together in a single pandas dataframe. Each row is the evaluation of one retrieval system over one particular dataset. The dataframe will be saved to csv/df_eval.csv and loaded from there, as long as the file exists."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "18b03049-d66f-4b42-8643-6cc7abdd5a83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>dataset</th>\n",
       "      <th>team</th>\n",
       "      <th>run_id</th>\n",
       "      <th>software</th>\n",
       "      <th>is_upload</th>\n",
       "      <th>is_docker</th>\n",
       "      <th>nDCG@10</th>\n",
       "      <th>RR</th>\n",
       "      <th>P@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-14-10-11</td>\n",
       "      <td>BM25 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.510</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-14-12-44</td>\n",
       "      <td>BM25+RM3 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.488</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-18-22-45</td>\n",
       "      <td>DFR_BM25 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.508</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-18-22-16</td>\n",
       "      <td>DFIC (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.522</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-18-22-58</td>\n",
       "      <td>DFRee (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.453</td>\n",
       "      <td>0.840</td>\n",
       "      <td>0.569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-07-20-26-06</td>\n",
       "      <td>ANCE Base Cosine (tira-ir-starter-beir)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.357</td>\n",
       "      <td>0.602</td>\n",
       "      <td>0.374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-07-19-46-45</td>\n",
       "      <td>MonoBERT Large (tira-ir-starter-gygaggle)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.438</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-07-20-26-56</td>\n",
       "      <td>ANCE Base Dot (tira-ir-starter-beir)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.368</td>\n",
       "      <td>0.578</td>\n",
       "      <td>0.388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-07-18-26-34</td>\n",
       "      <td>MonoT5 Base (tira-ir-starter-gygaggle)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.451</td>\n",
       "      <td>0.637</td>\n",
       "      <td>0.496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-08-10-34-43</td>\n",
       "      <td>ColBERT Re-Rank (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.425</td>\n",
       "      <td>0.665</td>\n",
       "      <td>0.462</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              task                                   dataset             team  \\\n",
       "0    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "1    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "2    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "3    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "4    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "..             ...                                       ...              ...   \n",
       "235  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "236  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "237  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "238  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "239  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "\n",
       "                  run_id                                     software  \\\n",
       "0    2023-01-07-14-10-11             BM25 (tira-ir-starter-pyterrier)   \n",
       "1    2023-01-07-14-12-44         BM25+RM3 (tira-ir-starter-pyterrier)   \n",
       "2    2023-01-07-18-22-45         DFR_BM25 (tira-ir-starter-pyterrier)   \n",
       "3    2023-01-07-18-22-16             DFIC (tira-ir-starter-pyterrier)   \n",
       "4    2023-01-07-18-22-58            DFRee (tira-ir-starter-pyterrier)   \n",
       "..                   ...                                          ...   \n",
       "235  2023-02-07-20-26-06      ANCE Base Cosine (tira-ir-starter-beir)   \n",
       "236  2023-02-07-19-46-45    MonoBERT Large (tira-ir-starter-gygaggle)   \n",
       "237  2023-02-07-20-26-56         ANCE Base Dot (tira-ir-starter-beir)   \n",
       "238  2023-02-07-18-26-34       MonoT5 Base (tira-ir-starter-gygaggle)   \n",
       "239  2023-02-08-10-34-43  ColBERT Re-Rank (tira-ir-starter-pyterrier)   \n",
       "\n",
       "     is_upload  is_docker  nDCG@10     RR   P@10  \n",
       "0        False       True    0.510  0.935  0.747  \n",
       "1        False       True    0.488  0.920  0.715  \n",
       "2        False       True    0.508  0.938  0.741  \n",
       "3        False       True    0.522  0.920  0.713  \n",
       "4        False       True    0.453  0.840  0.569  \n",
       "..         ...        ...      ...    ...    ...  \n",
       "235      False       True    0.357  0.602  0.374  \n",
       "236      False       True    0.438  0.647  0.478  \n",
       "237      False       True    0.368  0.578  0.388  \n",
       "238      False       True    0.451  0.637  0.496  \n",
       "239      False       True    0.425  0.665  0.462  \n",
       "\n",
       "[240 rows x 10 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_file = Path(f'csv/df_eval.csv')\n",
    "\n",
    "if not output_file.exists():\n",
    "    df_eval = []\n",
    "    for dataset in tqdm(tira.datasets(TASK)):\n",
    "        df_eval += [tira.evaluations(task=TASK, dataset=dataset)]\n",
    "    df_eval = pd.concat(df_eval)\n",
    "    \n",
    "    output_file.parent.mkdir(exist_ok=True, parents=True)\n",
    "    df_eval.to_csv(output_file, index=False)\n",
    "\n",
    "else:\n",
    "    df_eval = pd.read_csv(output_file)\n",
    "\n",
    "df_eval"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ae92e8f7",
   "metadata": {},
   "source": [
    "### Clean dataframe from duplicate evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d4ff7b69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>dataset</th>\n",
       "      <th>team</th>\n",
       "      <th>run_id</th>\n",
       "      <th>software</th>\n",
       "      <th>is_upload</th>\n",
       "      <th>is_docker</th>\n",
       "      <th>nDCG@10</th>\n",
       "      <th>RR</th>\n",
       "      <th>P@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-14-10-11</td>\n",
       "      <td>BM25 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.510</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-14-12-44</td>\n",
       "      <td>BM25+RM3 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.488</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-18-22-45</td>\n",
       "      <td>DFR_BM25 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.508</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-18-22-16</td>\n",
       "      <td>DFIC (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.522</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-01-07-18-22-58</td>\n",
       "      <td>DFRee (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.453</td>\n",
       "      <td>0.840</td>\n",
       "      <td>0.569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-07-20-26-06</td>\n",
       "      <td>ANCE Base Cosine (tira-ir-starter-beir)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.357</td>\n",
       "      <td>0.602</td>\n",
       "      <td>0.374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-07-19-46-45</td>\n",
       "      <td>MonoBERT Large (tira-ir-starter-gygaggle)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.438</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-07-20-26-56</td>\n",
       "      <td>ANCE Base Dot (tira-ir-starter-beir)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.368</td>\n",
       "      <td>0.578</td>\n",
       "      <td>0.388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-07-18-26-34</td>\n",
       "      <td>MonoT5 Base (tira-ir-starter-gygaggle)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.451</td>\n",
       "      <td>0.637</td>\n",
       "      <td>0.496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>wapo-v2-trec-core-2018-20230107-training</td>\n",
       "      <td>tira-ir-starter</td>\n",
       "      <td>2023-02-08-10-34-43</td>\n",
       "      <td>ColBERT Re-Rank (tira-ir-starter-pyterrier)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.425</td>\n",
       "      <td>0.665</td>\n",
       "      <td>0.462</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>230 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              task                                   dataset             team  \\\n",
       "0    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "1    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "2    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "3    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "4    ir-benchmarks            antique-test-20230107-training  tira-ir-starter   \n",
       "..             ...                                       ...              ...   \n",
       "235  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "236  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "237  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "238  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "239  ir-benchmarks  wapo-v2-trec-core-2018-20230107-training  tira-ir-starter   \n",
       "\n",
       "                  run_id                                     software  \\\n",
       "0    2023-01-07-14-10-11             BM25 (tira-ir-starter-pyterrier)   \n",
       "1    2023-01-07-14-12-44         BM25+RM3 (tira-ir-starter-pyterrier)   \n",
       "2    2023-01-07-18-22-45         DFR_BM25 (tira-ir-starter-pyterrier)   \n",
       "3    2023-01-07-18-22-16             DFIC (tira-ir-starter-pyterrier)   \n",
       "4    2023-01-07-18-22-58            DFRee (tira-ir-starter-pyterrier)   \n",
       "..                   ...                                          ...   \n",
       "235  2023-02-07-20-26-06      ANCE Base Cosine (tira-ir-starter-beir)   \n",
       "236  2023-02-07-19-46-45    MonoBERT Large (tira-ir-starter-gygaggle)   \n",
       "237  2023-02-07-20-26-56         ANCE Base Dot (tira-ir-starter-beir)   \n",
       "238  2023-02-07-18-26-34       MonoT5 Base (tira-ir-starter-gygaggle)   \n",
       "239  2023-02-08-10-34-43  ColBERT Re-Rank (tira-ir-starter-pyterrier)   \n",
       "\n",
       "     is_upload  is_docker  nDCG@10     RR   P@10  \n",
       "0        False       True    0.510  0.935  0.747  \n",
       "1        False       True    0.488  0.920  0.715  \n",
       "2        False       True    0.508  0.938  0.741  \n",
       "3        False       True    0.522  0.920  0.713  \n",
       "4        False       True    0.453  0.840  0.569  \n",
       "..         ...        ...      ...    ...    ...  \n",
       "235      False       True    0.357  0.602  0.374  \n",
       "236      False       True    0.438  0.647  0.478  \n",
       "237      False       True    0.368  0.578  0.388  \n",
       "238      False       True    0.451  0.637  0.496  \n",
       "239      False       True    0.425  0.665  0.462  \n",
       "\n",
       "[230 rows x 10 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# df_eval = df_eval[df_eval['dataset'] == dataset].sort_values('nDCG@10', ascending=False)\n",
    "df_eval = df_eval.drop_duplicates(subset=['dataset', 'software'], keep=False)\n",
    "df_eval"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "077c06d8",
   "metadata": {},
   "source": [
    "### Get sets of all datasets and retrieval systems used in the evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "92da67a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['antique-test-20230107-training',\n",
       "       'clueweb09-en-trec-web-2009-20230107-training',\n",
       "       'clueweb09-en-trec-web-2010-20230107-training',\n",
       "       'clueweb09-en-trec-web-2011-20230107-training',\n",
       "       'clueweb09-en-trec-web-2012-20230107-training',\n",
       "       'clueweb12-trec-web-2013-20230107-training',\n",
       "       'clueweb12-trec-web-2014-20230107-training',\n",
       "       'cord19-fulltext-trec-covid-20230107-training',\n",
       "       'cranfield-20230107-training',\n",
       "       'medline-2004-trec-genomics-2004-20230107-training',\n",
       "       'msmarco-passage-trec-dl-2019-judged-20230107-training',\n",
       "       'vaswani-20230107-training',\n",
       "       'wapo-v2-trec-core-2018-20230107-training'], dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATASETS = df_eval['dataset'].unique()\n",
    "DATASETS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b367518f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['BM25 (tira-ir-starter-pyterrier)',\n",
       "       'BM25+RM3 (tira-ir-starter-pyterrier)',\n",
       "       'DFR_BM25 (tira-ir-starter-pyterrier)',\n",
       "       'DFIC (tira-ir-starter-pyterrier)',\n",
       "       'DFRee (tira-ir-starter-pyterrier)',\n",
       "       'DFIZ (tira-ir-starter-pyterrier)',\n",
       "       'DFReeKLIM (tira-ir-starter-pyterrier)',\n",
       "       'DirichletLM (tira-ir-starter-pyterrier)',\n",
       "       'DLH (tira-ir-starter-pyterrier)',\n",
       "       'DPH (tira-ir-starter-pyterrier)',\n",
       "       'Hiemstra_LM (tira-ir-starter-pyterrier)',\n",
       "       'IFB2 (tira-ir-starter-pyterrier)',\n",
       "       'In_expB2 (tira-ir-starter-pyterrier)',\n",
       "       'TF_IDF (tira-ir-starter-pyterrier)',\n",
       "       'XSqrA_M (tira-ir-starter-pyterrier)',\n",
       "       'InL2 (tira-ir-starter-pyterrier)',\n",
       "       'PL2 (tira-ir-starter-pyterrier)',\n",
       "       'LGD (tira-ir-starter-pyterrier)',\n",
       "       'In_expC2 (tira-ir-starter-pyterrier)',\n",
       "       'InB2 (tira-ir-starter-pyterrier)',\n",
       "       'Js_KLs (tira-ir-starter-pyterrier)',\n",
       "       'BM25 Re-Rank (tira-ir-starter-pyterrier)',\n",
       "       'DFIC Re-Rank (tira-ir-starter-pyterrier)',\n",
       "       'MonoBERT Large (tira-ir-starter-gygaggle)',\n",
       "       'MonoT5 Base (tira-ir-starter-gygaggle)',\n",
       "       'ANCE Base Cosine (tira-ir-starter-beir)',\n",
       "       'ColBERT Re-Rank (tira-ir-starter-pyterrier)', 'ChatNoir',\n",
       "       'ANCE Base Dot (tira-ir-starter-beir)'], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SOFTWARES = df_eval['software'].unique()\n",
    "SOFTWARES"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "96b48193-8fb8-4700-af3a-056de214d530",
   "metadata": {},
   "source": [
    "### Download the run files of every evaluation\n",
    "Loads the runs file of each evaluation of the TIRA task from TIRA, if the file does not already exist locally.  Processes the file data to an appropriate format and saves it to runs/{dataset}/run_{dataset}_{software}.txt."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7f44bd59-135e-4757-be29-f55add7df4f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 13/13 [00:00<00:00, 75.91it/s]\n"
     ]
    }
   ],
   "source": [
    "for dataset in tqdm(DATASETS):\n",
    "    for software in SOFTWARES:\n",
    "        if not df_eval[(df_eval.dataset == dataset) & (df_eval.software == software)].empty:\n",
    "            output_file = Path(f'runs/{dataset}/run_{dataset}_{software}.txt')\n",
    "            if not output_file.exists():\n",
    "                run = tira.download_run(TASK, dataset, software)\n",
    "                output_file.parent.mkdir(exist_ok=True, parents=True)\n",
    "                output_file.write_text(run.to_string(header=False, index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4e0af8d-d1dc-4cf9-8f29-e1449ada1e41",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Map TIRA dataset names to coresponding ir_dataset IDs\n",
    "A dictionary to map the TIRA format of dataset names to the IDs of ir_datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "97521b4d-cca2-4e4f-9e44-da5e7134f83e",
   "metadata": {},
   "outputs": [],
   "source": [
    "IR_DATASET_IDS = {\n",
    "    'antique-test-20230107-training': 'antique/test',\n",
    "    'clueweb09-en-trec-web-2009-20230107-training': 'clueweb09/en/trec-web-2009',\n",
    "    'clueweb09-en-trec-web-2010-20230107-training': 'clueweb09/en/trec-web-2010',\n",
    "    'clueweb09-en-trec-web-2011-20230107-training': 'clueweb09/en/trec-web-2011',\n",
    "    'clueweb09-en-trec-web-2012-20230107-training': 'clueweb09/en/trec-web-2012',\n",
    "    'clueweb12-trec-web-2013-20230107-training': 'clueweb12/trec-web-2013',\n",
    "    'clueweb12-trec-web-2014-20230107-training': 'clueweb12/trec-web-2013',\n",
    "    'cord19-fulltext-trec-covid-20230107-training': 'cord19/fulltext/trec-covid',\n",
    "    'cranfield-20230107-training': 'cranfield',\n",
    "    'medline-2004-trec-genomics-2004-20230107-training': 'medline/2004/trec-genomics-2004',\n",
    "    'msmarco-passage-trec-dl-2019-judged-20230107-training': 'msmarco-passage/trec-dl-2019/judged',\n",
    "    'vaswani-20230107-training': 'vaswani',\n",
    "    'wapo-v2-trec-core-2018-20230107-training': 'wapo/v2/trec-core-2018'\n",
    "}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "12fd8f59-8f9e-4e7e-8cbc-912def8ceb19",
   "metadata": {},
   "source": [
    "### Load qrels via ir_datasets\n",
    "Downloads the qrels of each dataset with ir_datasets, if the file does not already exists locally. Processes the data to an appropriate format and saves it to qrels/qrels_{dataset}.txt."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fcfd2151-09fa-4018-b453-b780e327343c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 13/13 [00:00<00:00, 31961.28it/s]\n"
     ]
    }
   ],
   "source": [
    "def map_qrel(qrel: tuple) -> str:\n",
    "    iteration = getattr(qrel, 'iteration', 0)\n",
    "    return f\"{qrel.query_id} {iteration} {qrel.doc_id} {qrel.relevance}\"\n",
    "\n",
    "def write_qrels_to_file(qrels, path):\n",
    "    output_file.parent.mkdir(exist_ok=True, parents=True)\n",
    "    with output_file.open('wt') as f:\n",
    "        f.writelines('%s\\n' % qrel for qrel in qrels)\n",
    "\n",
    "for dataset in tqdm(DATASETS):\n",
    "    output_file = Path(f'qrels/qrels_{dataset}.txt')\n",
    "    if not output_file.exists():\n",
    "        ir_ds = ir_datasets.load(IR_DATASET_IDS[dataset])\n",
    "        qrels = (map_qrel(qrel) for qrel in ir_ds.qrels_iter())\n",
    "        write_qrels_to_file(qrels, output_file)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e6873cee-3503-4f4b-be6d-01ce1e6de41a",
   "metadata": {},
   "source": [
    "### Function to evaluate the replication of results of retrieval systems on another dataset \n",
    "Uses repro_eval to evaluate replicability tests of a tuple of retrieval systems across datasets.  \n",
    "Is frequently called by the two functions further down to evaluate the dataset wide replicability of retrieval systems."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "87af7c0a-241e-4494-a310-5a83ec603d50",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_replicability(qrels_orig, run_b_orig, run_a_orig, run_b_rpl, run_a_rpl, qrels_rpl, measures):\n",
    "    rpl_eval = RplEvaluator(qrel_orig_path=qrels_orig,\n",
    "                        run_b_orig_path=run_b_orig,\n",
    "                        run_a_orig_path=run_a_orig,\n",
    "                        run_b_rep_path=run_b_rpl,\n",
    "                        run_a_rep_path=run_a_rpl,\n",
    "                        qrel_rpl_path=qrels_rpl)\n",
    "    rpl_eval.trim()\n",
    "    rpl_eval.evaluate()\n",
    "    \n",
    "    er = rpl_eval.er()\n",
    "    dri = rpl_eval.dri()\n",
    "    p = rpl_eval.ttest()\n",
    "    \n",
    "    metrics = {}\n",
    "\n",
    "    for measure in measures:\n",
    "        metrics[measure] = {\n",
    "            'measure': measure,\n",
    "            'er': er[measure],\n",
    "            'dri': dri[measure],\n",
    "            'p_base': p.get('baseline').get(measure),\n",
    "            'p_adv': p.get('advanced').get(measure),\n",
    "        }\n",
    "\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a2925c4",
   "metadata": {},
   "source": [
    "Helper functions to write evaluations to files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8de7a5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_to_csv(timestamp, eval_type, evaluation):\n",
    "    output_file = Path(f'csv/eval_{eval_type}_{timestamp}.csv')\n",
    "    output_file.parent.mkdir(exist_ok=True, parents=True)\n",
    "    \n",
    "    df = pd.DataFrame(evaluation)\n",
    "    df = df.transpose()\n",
    "    df.to_csv(output_file, index=False)\n",
    "\n",
    "\n",
    "def write_to_json(timestamp, eval_type, evaluation):\n",
    "    output_file_json = Path(f'json/eval_{eval_type}_{timestamp}.json')\n",
    "    output_file_json.parent.mkdir(exist_ok=True, parents=True)\n",
    "    with output_file_json.open('w') as f:\n",
    "        json.dump(evaluation, f, indent=4)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ec431abb",
   "metadata": {},
   "source": [
    "Values to use for upcoming replicability evaluation processes:\n",
    "\n",
    "QRELS -> list of paths to all qrels files\n",
    "\n",
    "RUNS -> list of paths to all run files\n",
    "\n",
    "MEASURES -> list of all measures to measure in evaluation\n",
    "\n",
    "DATASET_TEST -> small test set of datasets for testing the evaluation process in a less time consuming process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "41e41443",
   "metadata": {},
   "outputs": [],
   "source": [
    "QRELS = glob.glob('qrels/*.txt')\n",
    "RUNS = glob.glob('runs/*/*.txt')\n",
    "\n",
    "MEASURES = [\n",
    "    'ndcg_cut_10', \n",
    "    'P_10', \n",
    "    'recip_rank',\n",
    "]\n",
    "\n",
    "DATASETS_TEST = np.array([\n",
    "    'antique-test-20230107-training',\n",
    "    'cord19-fulltext-trec-covid-20230107-training',\n",
    "])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "6c3963a0",
   "metadata": {},
   "source": [
    "### Function to run a full scale evaluation of replicability across all datasets\n",
    "Evaluates the replicability of every tuple of evaluated retrieval systems on all datasets against all other datersets on which these tuples has been evaluated.\n",
    "\n",
    "This is a full scale evaluation with n² evaluations in respect to the number of retrieval systems per dataset as well as n² evaluations in respect to the number of datasets.\n",
    "\n",
    "The results will be written to json and csv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "70e2d9e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def full_replicability_evaluation(datasets=DATASETS):\n",
    "    run_orig_base = ''\n",
    "    run_orig_adv = ''\n",
    "\n",
    "    run_rpl_base = ''\n",
    "    run_rpl_adv = ''\n",
    "\n",
    "    qrels_orig = ''\n",
    "    qrels_rpl = ''\n",
    "\n",
    "    evals = {}\n",
    "\n",
    "    for dataset in tqdm(datasets):\n",
    "        # set qrels of target original dataset\n",
    "        qrels_orig = [x for x in QRELS if dataset in x][0]\n",
    "\n",
    "        # isolate all evaluations on target original dataset\n",
    "        df_dataset = df_eval[df_eval['dataset'] == dataset].sort_values('nDCG@10', ascending=False)\n",
    "        \n",
    "        # extract list of all softwares of target original dataset\n",
    "        softwares_dataset = df_dataset['software'].unique()\n",
    "\n",
    "        # extract list of all datasets for replicatation\n",
    "        datasets_rpl = np.delete(datasets, datasets==dataset)\n",
    "\n",
    "        for sw_base in softwares_dataset:\n",
    "            # set baseline run of target original dataset\n",
    "            run_orig_base = [x for x in RUNS if dataset in x and sw_base in x][0]\n",
    "\n",
    "            # extract list of all softwares to measure against\n",
    "            softwares_advanced = np.delete(softwares_dataset, softwares_dataset==sw_base)\n",
    "\n",
    "            for sw_adv in softwares_advanced:\n",
    "                # set advanced run of target original dataset\n",
    "                run_orig_adv = [x for x in RUNS if dataset in x and sw_adv in x][0]\n",
    "            \n",
    "                for dataset_rpl in datasets_rpl:\n",
    "                    # isolate all evaluations on target replication dataset\n",
    "                    df_dataset_rpl = df_eval[df_eval['dataset'] == dataset_rpl]\n",
    "\n",
    "                    # extract list of all softwares of target replication dataset\n",
    "                    softwares_dataset_rpl = df_dataset_rpl['software'].unique()\n",
    "\n",
    "                    if sw_base in softwares_dataset_rpl and sw_adv in softwares_dataset_rpl:\n",
    "                        # set qrels of target original dataset\n",
    "                        qrels_rpl = [x for x in QRELS if dataset_rpl in x][0]\n",
    "\n",
    "                        # set baseline run of target replication dataset\n",
    "                        run_rpl_base = [x for x in RUNS if dataset_rpl in x and sw_base in x][0]\n",
    "\n",
    "                        # set baseline run of target replication dataset\n",
    "                        run_rpl_adv = [x for x in RUNS if dataset_rpl in x and sw_adv in x][0]\n",
    "\n",
    "                        metrics = evaluate_replicability(qrels_orig, run_orig_base, run_orig_adv, run_rpl_base, run_rpl_adv, qrels_rpl, MEASURES)\n",
    "\n",
    "                        for measure in MEASURES:\n",
    "                            evals[index] = {\n",
    "                                'dataset_orig': dataset,\n",
    "                                'dataset_rpl': dataset_rpl,\n",
    "                                'system_base': softwares_dataset[i],\n",
    "                                'system_adv': sw_adv,\n",
    "                                'measure': metrics[measure]['measure'],\n",
    "                                'er': metrics[measure]['er'],\n",
    "                                'dri': metrics[measure]['dri'],\n",
    "                                'p_base': metrics[measure]['p_base'],\n",
    "                                'p_adv': metrics[measure]['p_adv']\n",
    "                            }   \n",
    "                            index += 1\n",
    "\n",
    "\n",
    "    timestamp = datetime.now().strftime('%Y-%m-%d-%H-%M')\n",
    "    write_to_csv(timestamp, 'full', evals)\n",
    "    write_to_json(timestamp, 'full', evals)\n",
    "\n",
    "    return evals"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "f05281fd",
   "metadata": {},
   "source": [
    "### Function to run a binomical scale evaluation of replicability across all datasets\n",
    "Evaluates the replicability of ordered tuples of evaluated retrieval systems on all datasets against all other datersets on which these tuples has been evaluated.\n",
    "\n",
    "These tuples are ordered by their nDCG@10 value, where one retrieval system functions as a baseline against an advanced retrieval system. \n",
    "\n",
    "The scale of this evaluation is binomical in respect to the number of retrieval systems per dataset. It still is in the scale of n² in respect to the number of datasets.\n",
    "\n",
    "The results will be written to json and csv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "1506f40b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def binomial_replicability_evaluation(datasets=DATASETS):\n",
    "    run_orig_base = ''\n",
    "    run_orig_adv = ''\n",
    "    run_rpl_base = ''\n",
    "    run_rpl_adv = ''\n",
    "    qrels_orig = ''\n",
    "    qrels_rpl = ''\n",
    "\n",
    "    evals = {}\n",
    "    index = 0\n",
    "\n",
    "    # loop through all datasets\n",
    "    for dataset in tqdm(datasets):\n",
    "        # set qrels of target original dataset\n",
    "        qrels_orig = [x for x in QRELS if dataset in x][0]\n",
    "\n",
    "        # isolate all evaluations on target original dataset\n",
    "        df_dataset = df_eval[df_eval['dataset'] == dataset].sort_values('nDCG@10', ascending=True)\n",
    "        \n",
    "        # extract list of all softwares used on target original dataset\n",
    "        softwares_dataset = df_dataset['software'].unique()\n",
    "\n",
    "        # extract list of all datasets for replicatation\n",
    "        datasets_rpl = np.delete(datasets, datasets==dataset)\n",
    "\n",
    "        for i in range(len(softwares_dataset)):\n",
    "            # set baseline run of target original dataset\n",
    "            run_orig_base = [x for x in RUNS if dataset in x and softwares_dataset[i] in x][0]\n",
    "\n",
    "            # extract list of all softwares to measure against\n",
    "            softwares_advanced = softwares_dataset[i+1:]\n",
    "\n",
    "            for sw_adv in softwares_advanced:\n",
    "                # set advanced run of target original dataset\n",
    "                run_orig_adv = [x for x in RUNS if dataset in x and sw_adv in x][0]\n",
    "            \n",
    "                for dataset_rpl in datasets_rpl:\n",
    "                    # isolate all evaluations on target replication dataset\n",
    "                    df_dataset_rpl = df_eval[df_eval['dataset'] == dataset_rpl]\n",
    "\n",
    "                    # extract list of all softwares of target replication dataset\n",
    "                    softwares_dataset_rpl = df_dataset_rpl['software'].unique()\n",
    "\n",
    "                    if softwares_dataset[i] in softwares_dataset_rpl and sw_adv in softwares_dataset_rpl:\n",
    "                        # set qrels of target original dataset\n",
    "                        qrels_rpl = [x for x in QRELS if dataset_rpl in x][0]\n",
    "\n",
    "                        # set baseline run of target replication dataset\n",
    "                        run_rpl_base = [x for x in RUNS if dataset_rpl in x and softwares_dataset[i] in x][0]\n",
    "\n",
    "                        # set baseline run of target replication dataset\n",
    "                        run_rpl_adv = [x for x in RUNS if dataset_rpl in x and sw_adv in x][0]\n",
    "\n",
    "                        metrics = evaluate_replicability(qrels_orig, run_orig_base, run_orig_adv, run_rpl_base, run_rpl_adv, qrels_rpl, MEASURES)\n",
    "\n",
    "                        for measure in MEASURES:\n",
    "                            evals[index] = {\n",
    "                                'dataset_orig': dataset,\n",
    "                                'dataset_rpl': dataset_rpl,\n",
    "                                'system_base': softwares_dataset[i],\n",
    "                                'system_adv': sw_adv,\n",
    "                                'measure': metrics[measure]['measure'],\n",
    "                                'er': metrics[measure]['er'],\n",
    "                                'dri': metrics[measure]['dri'],\n",
    "                                'p_base': metrics[measure]['p_base'],\n",
    "                                'p_adv': metrics[measure]['p_adv']\n",
    "                            }   \n",
    "                            index += 1\n",
    "\n",
    "\n",
    "    timestamp = datetime.now().strftime('%Y-%m-%d-%H-%M')\n",
    "    write_to_csv(timestamp, 'binomial', evals)\n",
    "    write_to_json(timestamp, 'binomial', evals)\n",
    "\n",
    "    return evals\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "8e3a6e6d",
   "metadata": {},
   "source": [
    "Start replicability evaluation and print the resulting dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "f8d7153f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2 [00:00<?, ?it/s]/home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages/repro_eval/measure/overall_effects.py:142: RuntimeWarning: divide by zero encountered in double_scalars\n",
      "  yield measure, (mean - mean_scores_b.get(measure)) / mean_scores_b.get(measure)\n",
      "/home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages/repro_eval/measure/overall_effects.py:142: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  yield measure, (mean - mean_scores_b.get(measure)) / mean_scores_b.get(measure)\n",
      "/home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages/repro_eval/measure/statistics.py:32: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  yield measure, ttest_ind(scores, topic_scores_rep.get(measure)).pvalue\n",
      "/home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages/repro_eval/measure/overall_effects.py:82: RuntimeWarning: divide by zero encountered in double_scalars\n",
      "  yield measure, value / mi_orig.get(measure)\n",
      "/home/simon/anaconda3/envs/bachelor/lib/python3.10/site-packages/repro_eval/measure/overall_effects.py:82: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  yield measure, value / mi_orig.get(measure)\n",
      "100%|██████████| 2/2 [10:55<00:00, 327.91s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset_orig</th>\n",
       "      <th>dataset_rpl</th>\n",
       "      <th>system_base</th>\n",
       "      <th>system_adv</th>\n",
       "      <th>measure</th>\n",
       "      <th>er</th>\n",
       "      <th>dri</th>\n",
       "      <th>p_base</th>\n",
       "      <th>p_adv</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>DirichletLM (tira-ir-starter-pyterrier)</td>\n",
       "      <td>XSqrA_M (tira-ir-starter-pyterrier)</td>\n",
       "      <td>ndcg_cut_10</td>\n",
       "      <td>1.649898</td>\n",
       "      <td>-0.020232</td>\n",
       "      <td>0.000052</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>DirichletLM (tira-ir-starter-pyterrier)</td>\n",
       "      <td>XSqrA_M (tira-ir-starter-pyterrier)</td>\n",
       "      <td>P_10</td>\n",
       "      <td>2.241509</td>\n",
       "      <td>-0.084049</td>\n",
       "      <td>0.001181</td>\n",
       "      <td>0.000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>DirichletLM (tira-ir-starter-pyterrier)</td>\n",
       "      <td>XSqrA_M (tira-ir-starter-pyterrier)</td>\n",
       "      <td>recip_rank</td>\n",
       "      <td>0.751698</td>\n",
       "      <td>0.043927</td>\n",
       "      <td>0.890237</td>\n",
       "      <td>0.681235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>DirichletLM (tira-ir-starter-pyterrier)</td>\n",
       "      <td>ANCE Base Cosine (tira-ir-starter-beir)</td>\n",
       "      <td>ndcg_cut_10</td>\n",
       "      <td>0.926459</td>\n",
       "      <td>0.108797</td>\n",
       "      <td>0.000052</td>\n",
       "      <td>0.000042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>DirichletLM (tira-ir-starter-pyterrier)</td>\n",
       "      <td>ANCE Base Cosine (tira-ir-starter-beir)</td>\n",
       "      <td>P_10</td>\n",
       "      <td>2.35125</td>\n",
       "      <td>-0.056948</td>\n",
       "      <td>0.001181</td>\n",
       "      <td>0.000004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1795</th>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>In_expB2 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>MonoT5 Base (tira-ir-starter-gygaggle)</td>\n",
       "      <td>P_10</td>\n",
       "      <td>-5.20202</td>\n",
       "      <td>0.334073</td>\n",
       "      <td>0.253824</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1796</th>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>In_expB2 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>MonoT5 Base (tira-ir-starter-gygaggle)</td>\n",
       "      <td>recip_rank</td>\n",
       "      <td>-1.1763</td>\n",
       "      <td>0.203697</td>\n",
       "      <td>0.000122</td>\n",
       "      <td>0.386362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1797</th>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>In_expC2 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>MonoT5 Base (tira-ir-starter-gygaggle)</td>\n",
       "      <td>ndcg_cut_10</td>\n",
       "      <td>0.136684</td>\n",
       "      <td>0.076017</td>\n",
       "      <td>0.000222</td>\n",
       "      <td>0.000002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1798</th>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>In_expC2 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>MonoT5 Base (tira-ir-starter-gygaggle)</td>\n",
       "      <td>P_10</td>\n",
       "      <td>-4.608586</td>\n",
       "      <td>0.310467</td>\n",
       "      <td>0.566708</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1799</th>\n",
       "      <td>cord19-fulltext-trec-covid-20230107-training</td>\n",
       "      <td>antique-test-20230107-training</td>\n",
       "      <td>In_expC2 (tira-ir-starter-pyterrier)</td>\n",
       "      <td>MonoT5 Base (tira-ir-starter-gygaggle)</td>\n",
       "      <td>recip_rank</td>\n",
       "      <td>-1.395731</td>\n",
       "      <td>0.192923</td>\n",
       "      <td>0.000238</td>\n",
       "      <td>0.386362</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1800 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      dataset_orig  \\\n",
       "0                   antique-test-20230107-training   \n",
       "1                   antique-test-20230107-training   \n",
       "2                   antique-test-20230107-training   \n",
       "3                   antique-test-20230107-training   \n",
       "4                   antique-test-20230107-training   \n",
       "...                                            ...   \n",
       "1795  cord19-fulltext-trec-covid-20230107-training   \n",
       "1796  cord19-fulltext-trec-covid-20230107-training   \n",
       "1797  cord19-fulltext-trec-covid-20230107-training   \n",
       "1798  cord19-fulltext-trec-covid-20230107-training   \n",
       "1799  cord19-fulltext-trec-covid-20230107-training   \n",
       "\n",
       "                                       dataset_rpl  \\\n",
       "0     cord19-fulltext-trec-covid-20230107-training   \n",
       "1     cord19-fulltext-trec-covid-20230107-training   \n",
       "2     cord19-fulltext-trec-covid-20230107-training   \n",
       "3     cord19-fulltext-trec-covid-20230107-training   \n",
       "4     cord19-fulltext-trec-covid-20230107-training   \n",
       "...                                            ...   \n",
       "1795                antique-test-20230107-training   \n",
       "1796                antique-test-20230107-training   \n",
       "1797                antique-test-20230107-training   \n",
       "1798                antique-test-20230107-training   \n",
       "1799                antique-test-20230107-training   \n",
       "\n",
       "                                  system_base  \\\n",
       "0     DirichletLM (tira-ir-starter-pyterrier)   \n",
       "1     DirichletLM (tira-ir-starter-pyterrier)   \n",
       "2     DirichletLM (tira-ir-starter-pyterrier)   \n",
       "3     DirichletLM (tira-ir-starter-pyterrier)   \n",
       "4     DirichletLM (tira-ir-starter-pyterrier)   \n",
       "...                                       ...   \n",
       "1795     In_expB2 (tira-ir-starter-pyterrier)   \n",
       "1796     In_expB2 (tira-ir-starter-pyterrier)   \n",
       "1797     In_expC2 (tira-ir-starter-pyterrier)   \n",
       "1798     In_expC2 (tira-ir-starter-pyterrier)   \n",
       "1799     In_expC2 (tira-ir-starter-pyterrier)   \n",
       "\n",
       "                                   system_adv      measure        er  \\\n",
       "0         XSqrA_M (tira-ir-starter-pyterrier)  ndcg_cut_10  1.649898   \n",
       "1         XSqrA_M (tira-ir-starter-pyterrier)         P_10  2.241509   \n",
       "2         XSqrA_M (tira-ir-starter-pyterrier)   recip_rank  0.751698   \n",
       "3     ANCE Base Cosine (tira-ir-starter-beir)  ndcg_cut_10  0.926459   \n",
       "4     ANCE Base Cosine (tira-ir-starter-beir)         P_10   2.35125   \n",
       "...                                       ...          ...       ...   \n",
       "1795   MonoT5 Base (tira-ir-starter-gygaggle)         P_10  -5.20202   \n",
       "1796   MonoT5 Base (tira-ir-starter-gygaggle)   recip_rank   -1.1763   \n",
       "1797   MonoT5 Base (tira-ir-starter-gygaggle)  ndcg_cut_10  0.136684   \n",
       "1798   MonoT5 Base (tira-ir-starter-gygaggle)         P_10 -4.608586   \n",
       "1799   MonoT5 Base (tira-ir-starter-gygaggle)   recip_rank -1.395731   \n",
       "\n",
       "           dri    p_base     p_adv  \n",
       "0    -0.020232  0.000052       0.0  \n",
       "1    -0.084049  0.001181  0.000001  \n",
       "2     0.043927  0.890237  0.681235  \n",
       "3     0.108797  0.000052  0.000042  \n",
       "4    -0.056948  0.001181  0.000004  \n",
       "...        ...       ...       ...  \n",
       "1795  0.334073  0.253824       0.0  \n",
       "1796  0.203697  0.000122  0.386362  \n",
       "1797  0.076017  0.000222  0.000002  \n",
       "1798  0.310467  0.566708       0.0  \n",
       "1799  0.192923  0.000238  0.386362  \n",
       "\n",
       "[1800 rows x 9 columns]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evals = binomial_replicability_evaluation(DATASETS_TEST)\n",
    "\n",
    "df = pd.DataFrame(evals)\n",
    "df = df.transpose()\n",
    "\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bachelor",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "vscode": {
   "interpreter": {
    "hash": "68013ac4b2c3ad9074fa8e39ab5bbd1da9edfce42f4e0868dc95ea4beb50466e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}