notebooks/extract_results_controlled.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 1\n",
    "We extract the results for table 1 in this notebook. Replace the username with your Huggingface username if you are reproducing our results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "username = 'JasperDekoninck'\n",
    "\n",
    "base_path = f'../data/baselines/{username}'\n",
    "\n",
    "def bootstrap_tpr(scores, scores_false, n_bootstrap=1000):\n",
    "    values = []\n",
    "    thresholds = []\n",
    "    for _ in range(n_bootstrap):\n",
    "        random_indices = np.random.choice(range(len(scores)), len(scores), replace=True)\n",
    "        random_indices_false = np.random.choice(range(len(scores_false)), len(scores_false), replace=True)\n",
    "        scores_false_here = scores_false[random_indices_false]\n",
    "        scores_true = scores[random_indices]\n",
    "        threshold = np.sort(scores_false_here)[int(len(scores_false_here) * 0.99)]\n",
    "        thresholds.append(threshold)\n",
    "        tpr = (scores_true > threshold).mean()\n",
    "        values.append(tpr)\n",
    "    p = 1 - np.mean(np.array(values) > 0.01)\n",
    "    return p\n",
    "\n",
    "def sample_level_methods(df):\n",
    "    output_dict = dict()\n",
    "    output_dict['shi'] = df['topkmin']\n",
    "    output_dict['mireshgallah'] = - df['perplexity_output'] / df['perplexity_ref']\n",
    "    output_dict['yeom'] = - df['perplexity_output']\n",
    "    output_dict['carlini'] = - df['lowercase']\n",
    "    return output_dict\n",
    "\n",
    "def compute_tpr(scores, scores_false, fpr=0.01):\n",
    "    # compute the threshold\n",
    "    false_scores = np.sort(scores_false)\n",
    "    threshold = false_scores[int(len(false_scores) * (1-fpr))]\n",
    "    # compute the tpr\n",
    "    tpr = (scores > threshold).mean()\n",
    "    return tpr\n",
    "\n",
    "def detect(folder):\n",
    "    detection = {\n",
    "        'shi': 0,\n",
    "        'mireshgallah': 0,\n",
    "        'yeom': 0,\n",
    "        'carlini': 0\n",
    "    }\n",
    "    for file in os.listdir(folder):\n",
    "        if not file.endswith('.csv'):\n",
    "            continue\n",
    "        if 'False' not in file:\n",
    "            continue\n",
    "        df = pd.read_csv(os.path.join(folder, file))\n",
    "        methods = sample_level_methods(df)\n",
    "        df_true = pd.read_csv(os.path.join(folder, file.replace('False', 'True')))\n",
    "        methods_true = sample_level_methods(df_true)\n",
    "        for method, scores in methods.items():\n",
    "            scores_true = methods_true[method]\n",
    "            p = compute_tpr(np.array(scores), np.array(scores_true))\n",
    "            p = bootstrap_tpr(np.array(scores), np.array(scores_true))\n",
    "            if p < 0.05:\n",
    "                detection[method] += 1\n",
    "    return detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "detections = dict()\n",
    "for folder in os.listdir(base_path):\n",
    "    detection = detect(os.path.join(base_path, folder))\n",
    "    detections[folder.replace('contamination-models-', '')] = detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_path = '../tables'\n",
    "all_dfs = []\n",
    "for benchmark in ['gsm8k', 'mmlu', 'arc', 'hellaswag']:\n",
    "    df = pd.read_csv(os.path.join(base_path, f'{benchmark}_synthetic.csv'))\n",
    "    df = df[df['model'].apply(lambda x: 'contamination-models-' in x)]\n",
    "    df['model'] = df['model'].apply(lambda x: x.split('/')[-1].replace('contamination-models-', ''))\n",
    "    all_dfs.append(df)\n",
    "df_synthetic = pd.concat(all_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_path = '../tables'\n",
    "all_dfs = []\n",
    "for benchmark in ['gsm8k', 'mmlu', 'arc', 'hellaswag']:\n",
    "    df = pd.read_csv(os.path.join(base_path, f'{benchmark}_rephrase.csv'))\n",
    "    df = df[df['model'].apply(lambda x: 'contamination-models-' in x)]\n",
    "    df['model'] = df['model'].apply(lambda x: x.split('/')[-1].replace('contamination-models-', ''))\n",
    "    all_dfs.append(df)\n",
    "df_rephrase = pd.concat(all_dfs).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_synthetic['contaminated'] = np.logical_and(df_synthetic['score_model'] > df_synthetic['no_cont'], df_synthetic['score_model'] > 0.3)\n",
    "df_rephrase['contaminated'] = np.logical_and(df_rephrase['score_model'] > df_rephrase['no_cont'], df_rephrase['score_model'] > 0.3)\n",
    "df_synthetic['detected'] = df_synthetic['p_value'] < 0.05\n",
    "df_rephrase['detected'] = df_rephrase['p_value'] < 0.05\n",
    "rephrased_models = np.array(df_rephrase['model'].apply(lambda x: 'rephrase' not in x))\n",
    "df_rephrase['syntax_contaminated'] = np.logical_and(df_rephrase['contaminated'], rephrased_models)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "61\n"
     ]
    }
   ],
   "source": [
    "print(np.count_nonzero(df_synthetic['contaminated']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9836065573770492\n"
     ]
    }
   ],
   "source": [
    "print(np.count_nonzero(np.logical_and(df_synthetic['detected'], df_synthetic['contaminated'])) / np.count_nonzero(df_synthetic['contaminated']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8913043478260869\n"
     ]
    }
   ],
   "source": [
    "print(np.count_nonzero(np.logical_and(df_rephrase['detected'], df_rephrase['syntax_contaminated'])) / np.count_nonzero(df_rephrase['syntax_contaminated']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shi 43 0.7049180327868853\n",
      "mireshgallah 42 0.6885245901639344\n",
      "yeom 41 0.6721311475409836\n",
      "carlini 40 0.6557377049180327\n"
     ]
    }
   ],
   "source": [
    "df_methods = pd.DataFrame(detections).T\n",
    "\n",
    "# remove the models that are not contaminated\n",
    "df_methods = df_methods.loc[df_rephrase[df_rephrase['contaminated']]['model'].unique()]\n",
    "\n",
    "for method in df_methods.columns:\n",
    "    print(method, np.count_nonzero(df_methods[method]), np.count_nonzero(df_methods[method]) / np.count_nonzero(df_rephrase['contaminated']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shi 39 0.8478260869565217\n",
      "mireshgallah 35 0.7608695652173914\n",
      "yeom 36 0.782608695652174\n",
      "carlini 35 0.7608695652173914\n"
     ]
    }
   ],
   "source": [
    "df_methods = pd.DataFrame(detections).T\n",
    "\n",
    "# remove the models that are not contaminated\n",
    "df_methods = df_methods.loc[df_rephrase[df_rephrase['syntax_contaminated']]['model'].unique()]\n",
    "\n",
    "for method in df_methods.columns:\n",
    "    print(method, np.count_nonzero(df_methods[method]), np.count_nonzero(df_methods[method]) / np.count_nonzero(df_rephrase['syntax_contaminated']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets extrac the shi results\n",
    "path = '../code-contamination-detection/code-contamination-output'\n",
    "# recursive search for the shi results for all files called 'log.txt'\n",
    "import glob\n",
    "\n",
    "all_files = glob.glob(path + '/**/log.txt', recursive=True)\n",
    "results = []\n",
    "\n",
    "for file in all_files:\n",
    "    model_name = file.split('/')[-2].replace('contamination-models-', '')\n",
    "    with open(file, 'r') as f:\n",
    "        lines = f.readlines()\n",
    "        for line in lines:\n",
    "            if 'result' in line:\n",
    "                score = float(line.split(' ')[-1].strip())\n",
    "                results.append({'score_shi': score, 'model': model_name})\n",
    "                break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "shi_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge shi_df with the rephrase results\n",
    "df_shi = df_rephrase.merge(shi_df, on='model', how='left', suffixes=('', '_shi'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.21739130434782608\n"
     ]
    }
   ],
   "source": [
    "print(np.count_nonzero(np.logical_and(df_shi['score_shi'] > 0.85, df_shi['syntax_contaminated'])) / np.count_nonzero(df_shi['syntax_contaminated']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.16393442622950818\n"
     ]
    }
   ],
   "source": [
    "print(np.count_nonzero(np.logical_and(df_shi['score_shi'] > 0.85, df_shi['contaminated'])) / np.count_nonzero(df_shi['contaminated']))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "contamination",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}