UnsupervisedAllLangs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9a2b9bf4-a536-4598-bdde-5fa48562b9b7",
   "metadata": {},
   "source": [
    "# Select Model\n",
    "\n",
    "### For this notebook, the only supported method is Unsupervised method. The variable is only used to ensure code reuse and for ensuring file naming convention for later.\n",
    "\n",
    "### Please do not change the \"tqmodel\" variable value in this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "747987e3-5bc0-4877-ade6-09302af82a17",
   "metadata": {},
   "outputs": [],
   "source": [
    "tqmodel = \"unsupervised\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ca3bf2f-f9f2-4af0-bcc0-a6b23793034a",
   "metadata": {},
   "source": [
    "# Select Language\n",
    "\n",
    "### You will need to select one source language for Unsupervised QE method evaluation, at a time. \n",
    "### The source language can be specified with the variable \"source_lang\" defined below. \n",
    "### The valid values for this variable are:\n",
    "\n",
    "* ru   ->   (Russian)\n",
    "* ro   ->   (Romanian)\n",
    "* si   ->   (Sinhala)\n",
    "* et   ->   (Estonian)\n",
    "* ne   ->   (Nepali)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3773d4b-bdc4-4f63-8c33-2f5964f5cf30",
   "metadata": {},
   "outputs": [],
   "source": [
    "source_lang = \"si\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef70db2c-61fc-4732-bbdc-c3dca5cc6071",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import io, sys, os\n",
    "from functools import reduce\n",
    "import spacy\n",
    "import string\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "from spacy.lang.en import English\n",
    "import numpy as np\n",
    "import random\n",
    "import re\n",
    "nlp = English()\n",
    "tokenizer = nlp.tokenizer\n",
    "from tqdm.notebook import tqdm\n",
    "import plotly.express as px\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from datasets import load_metric\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "018e0eca-5af9-4a17-9fe1-416c8164edb3",
   "metadata": {},
   "source": [
    "### Obtaining human judgements from the WMT data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b96a069e-fff6-498d-930d-ce0703fe70f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trans_thres = []\n",
    "dfdev = pd.read_csv(f\"{source_lang}_en_data/dev.{source_lang}en.df.short.tsv\",sep='\\t')\n",
    "dftest = pd.read_csv(f\"{source_lang}_en_data/test20.{source_lang}en.df.short.tsv\", sep='\\t')\n",
    "frames = [dfdev, dftest]\n",
    "final = pd.concat(frames)\n",
    "comb = final.reset_index(drop=True)\n",
    "devtestComb = comb.drop(columns=['index', 'mean', 'z_scores', 'z_mean', 'model_scores', 'scores'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34442bf0-5f1b-4b66-8840-a33f1ad80a8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_orig_trans_score = devtestComb\n",
    "####### BELOW CODE CALCULATES THE AVERAGE SCORE FOR HUMAN DA AND SUBSTITUTES IT IN THE TRANS_SCORE COLUMN INSTEAD OF THE PREDICTED SCORE #########\n",
    "l = comb['scores'].tolist()\n",
    "newlist = []\n",
    "for sublist in l:\n",
    "    sublist = sublist.split(\", \")\n",
    "    sublist = [i.strip('[]') for i in sublist]\n",
    "    sublist = [float(i) for i in sublist]\n",
    "    newlist.append(sublist)\n",
    "\n",
    "newlistnparray = np.array(newlist)\n",
    "newlistmeans = np.mean(newlistnparray, axis=1)\n",
    "newlistmeans = newlistmeans/100\n",
    "newlistmeans = newlistmeans.tolist()\n",
    "df_orig_trans_score['trans_score_hum'] = newlistmeans\n",
    "####### ABOVE CODE CALCULATES THE AVERAGE SCORE FOR HUMAN DA AND SUBSTITUTES IT IN THE TRANS_SCORE COLUMN INSTEAD OF THE PREDICTED SCORE #########\n",
    "\n",
    "def trans_threshold(df, th=0.0):\n",
    "  trans_score_np = np.array(list(df['trans_score_hum'])) ## CHANGE HERE BASED ON WHICH COLUMN SHOULD THE THRESHOLDING BE APPLIED\n",
    "  trans_score_idx_ = np.argwhere(trans_score_np >= th)\n",
    "  trans_score_idx = trans_score_idx_.reshape(trans_score_idx_.shape[0])\n",
    "\n",
    "  return df.iloc[trans_score_idx]\n",
    "\n",
    "trans_np = np.array(list(df_orig_trans_score['trans_score_hum']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50d359df-82d3-4692-88ec-5b6edd214e02",
   "metadata": {},
   "source": [
    "### Unsupervised method definition for obtaining scores for all sentences initially."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1686e7fd-0c69-485d-ab1d-8ca98c4cadcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trans_thres = trans_threshold(df_orig_trans_score, 0.0) \n",
    "df_trans_thres['ref'] = df_trans_thres['translation']\n",
    "df_trans_thres['trans_score'] = df_trans_thres['trans_score_hum']\n",
    "df_trans_thres = df_trans_thres.drop(columns=['trans_score_hum'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3c1ef99-6194-46b2-896a-7fb8a63c470d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_metrics(*args, **kwargs):\n",
    "    assert len(args) == len(kwargs[\"corr\"])\n",
    "    assert len(args[0]) == len(args[1])\n",
    "    \n",
    "    output = []\n",
    "    \n",
    "    for i in range(len(args[0])):\n",
    "        value = 0\n",
    "        for sign, metric in zip(kwargs[\"corr\"],args):\n",
    "            if sign > 0:\n",
    "                value += np.exp(metric[i])\n",
    "            else:\n",
    "                value += np.exp(1-metric[i])\n",
    "        output.append(value)\n",
    "    return output\n",
    "\n",
    "bert_score_metric = load_metric('bertscore', keep_in_memory=False, cache_dir=sys.path[0])\n",
    "bert_score_model = 'xlm-roberta-base'\n",
    "cos_sim_model = SentenceTransformer('xlm-r-bert-base-nli-stsb-mean-tokens')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae50a67d-ec03-4b99-b477-990849a9adb7",
   "metadata": {},
   "source": [
    "### Applying threshold of 0.7 now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bed9dd02-0693-4c3d-9f92-1387bb75c90f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trans_thres = trans_threshold(df_orig_trans_score, 0.7) #Selection of threshold 0.7\n",
    "df_trans_thres['ref'] = df_trans_thres['translation']\n",
    "df_trans_thres['trans_score'] = df_trans_thres['trans_score_hum']\n",
    "df_trans_thres = df_trans_thres.drop(columns=['trans_score_hum'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b999458-14f1-4e6a-8d1e-0716e0391d2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_scores_orig(df, nTrials=1):\n",
    "    ## Need a temporary scoring function here. \n",
    "    scores = np.zeros((1, df.shape[0]))\n",
    "  \n",
    "    for i in tqdm(range(nTrials)):\n",
    "        embed_sent1 = cos_sim_model.encode(df['translation'].values.tolist(), convert_to_tensor=True)\n",
    "        embed_sent2 = cos_sim_model.encode(df['original'].values.tolist(), convert_to_tensor=True)\n",
    "        cos_sim = nn.CosineSimilarity(dim=1)(embed_sent1,embed_sent2)\n",
    "        cos_sim = (cos_sim -torch.min(cos_sim))/ (torch.max(cos_sim)-torch.min(cos_sim))\n",
    "        cos_scores = cos_sim.cpu().numpy()\n",
    "\n",
    "        bert_score_metric.add_batch(predictions=df['translation'].values.tolist(), references=df['original'].values.tolist())\n",
    "        score = bert_score_metric.compute(model_type=bert_score_model)\n",
    "        norm_score = (score[\"f1\"] - np.min(score[\"f1\"])) / (np.max(score[\"f1\"]) - np.min(score[\"f1\"]))\n",
    "        compound_metric = combine_metrics(cos_scores, norm_score, corr=[1, 1])\n",
    "        df_norm = pd.DataFrame()\n",
    "        df_norm['scores'] = compound_metric\n",
    "        df_norm['scores'] = (df_norm['scores'] - df_norm['scores'].min()) / (df_norm['scores'].max() - df_norm['scores'].min())\n",
    "        predictions = df_norm['scores'].values.tolist()\n",
    "    \n",
    "        scores[i] = predictions\n",
    "\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "849a6e82-ea95-4a55-a3db-462533247d73",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_orig=[]\n",
    "scores_orig = predict_scores_orig(df_trans_thres)\n",
    "df_trans_thres.drop(columns='trans_score')\n",
    "df_trans_thres['trans_score'] = scores_orig[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be943602-a1fd-477d-b756-7a9ab82a17e2",
   "metadata": {},
   "source": [
    "### Vocbulary generation from the preprocessed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "295ea350-7f03-47ae-809b-88b765be451c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocabulary:\n",
    "    def __init__(self, name):\n",
    "        PAD_token = 0   # Used for padding short sentences\n",
    "        SOS_token = 1   # Start-of-sentence token\n",
    "        EOS_token = 2   # End-of-sentence token\n",
    "        self.name = name\n",
    "        self.word2index = {}\n",
    "        self.word2count = {}\n",
    "        self.index2word = {PAD_token: \"PAD\", SOS_token: \"SOS\", EOS_token: \"EOS\"}\n",
    "        self.num_words = 3\n",
    "        self.num_sentences = 0\n",
    "        self.longest_sentence = 0\n",
    "\n",
    "    def add_word(self, word):\n",
    "        if word not in self.word2index:\n",
    "            # First entry of word into vocabulary\n",
    "            self.word2index[word] = self.num_words\n",
    "            self.word2count[word] = 1\n",
    "            self.index2word[self.num_words] = word\n",
    "            self.num_words += 1\n",
    "        else:\n",
    "            # Word exists; increase word count\n",
    "            self.word2count[word] += 1\n",
    "            \n",
    "    def add_sentence(self, sentence):\n",
    "        sentence_len = 0\n",
    "        for word in sentence.split(' '):\n",
    "            sentence_len += 1\n",
    "            self.add_word(word)\n",
    "        if sentence_len > self.longest_sentence:\n",
    "            # This is the longest sentence\n",
    "            self.longest_sentence = sentence_len\n",
    "        # Count the number of sentences\n",
    "        self.num_sentences += 1\n",
    "\n",
    "    def to_word(self, index):\n",
    "        return self.index2word[index]\n",
    "\n",
    "    def to_index(self, word):\n",
    "        return self.word2index[word]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c02aaf0c-95df-4e38-b12b-49e5bb60a7a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "voc = Vocabulary('corpusVocab')\n",
    "for sentence in devtestComb['translation']:\n",
    "    voc.add_sentence(sentence.translate(str.maketrans('', '', string.punctuation))) ## removing puntuations\n",
    "\n",
    "# vocab with stopwords \n",
    "vocab = []\n",
    "for word in range(voc.num_words):\n",
    "    vocab.append(voc.to_word(word))\n",
    "# vocab without stopwords \n",
    "vocab_wo_stopw = []\n",
    "for word in vocab:\n",
    "    if(word.lower() not in STOP_WORDS):\n",
    "        vocab_wo_stopw.append(word)\n",
    "\n",
    "len(vocab), len(vocab_wo_stopw)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd813305-c690-4276-8c2d-08dd5a9390f7",
   "metadata": {},
   "source": [
    "## Perturbations and Scoring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e7c0d19-ce04-430b-8fbd-62a300a3603d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_map1_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translation based on the removal of negation markers - \"not, no, and n't\"\n",
    "    print(\"Generating sentences for MAP1\")\n",
    "    new_sents_lol = [] \n",
    "    map1_orig_sents_lol = []\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        map1_orig_sents = []\n",
    "        for original, sentence in zip(df['original'], df['translation']):\n",
    "            tempSentVoc = sentence.split()\n",
    "\n",
    "            sentVoc = []\n",
    "            for token in tempSentVoc:\n",
    "                token = token.replace(\"n't\", \"\", 1)\n",
    "                sentVoc.append(token)\n",
    "            \n",
    "            sentVoc = [token for token in sentVoc if (token.lower() != \"not\" and token.lower() != \"no\")]\n",
    "            \n",
    "            \n",
    "            if(sentence != ' '.join(sentVoc)):\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(' '.join(sentVoc))\n",
    "                new_sents.append(tempL)\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(sentence)\n",
    "                map1_orig_sents.append(tempL)\n",
    "        new_sents_lol.append(new_sents)\n",
    "        map1_orig_sents_lol.append(map1_orig_sents)\n",
    "\n",
    "    return new_sents_lol, map1_orig_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d57352a2-5f42-48c6-8ffe-6b3ef6403c4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_map2_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on removal of random content words\n",
    "    print(\"Generating sentences for MAP2\")\n",
    "    puncts = list(set(string.punctuation))\n",
    "    new_sents_lol = [] \n",
    "    vocab_ = vocab_wo_stopw\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        for sentence in df['translation']:\n",
    "            sentVoc = [str(i) for i in tokenizer(sentence)]\n",
    "            sentVoc_wo_punct = []\n",
    "\n",
    "            for v in sentVoc:\n",
    "                if(v not in puncts):\n",
    "                    sentVoc_wo_punct.append(v)\n",
    "\n",
    "            sentVoc = sentVoc_wo_punct\n",
    "            \n",
    "            sentVoc_wo_stop_word = []\n",
    "            for v in sentVoc:\n",
    "                if(v.lower() not in STOP_WORDS):\n",
    "                    sentVoc_wo_stop_word.append(v)\n",
    "            \n",
    "            sentVoc = sentVoc_wo_stop_word\n",
    "       \n",
    "            new_word = \"\" \n",
    "            old_word = random.choice(sentVoc)\n",
    "          \n",
    "            flag = 0\n",
    "            while(flag == 0):\n",
    "                if(new_word not in sentVoc):\n",
    "                    new_sents.append(sentence.replace(old_word, new_word, 1))\n",
    "                    flag = 1\n",
    "                else:\n",
    "                    new_word = random.choice(vocab_)\n",
    "    \n",
    "        new_sents_lol.append(new_sents)\n",
    "    return new_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad724737-4668-428a-9754-acbae90b87fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_map3_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on duplication of a random content word\n",
    "    print(\"Generating sentences for MAP3\")\n",
    "    puncts = list(set(string.punctuation))\n",
    "    new_sents_lol = []\n",
    "    vocab_ = vocab_wo_stopw\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        for sentence in df['translation']:\n",
    "            sentVoc = [str(i) for i in tokenizer(sentence)]\n",
    "            sentVoc_wo_punct = []\n",
    "\n",
    "            for v in sentVoc:\n",
    "                if(v not in puncts):\n",
    "                    sentVoc_wo_punct.append(v)\n",
    "            sentVoc = sentVoc_wo_punct\n",
    "\n",
    "            sentVoc_wo_stop_word = []\n",
    "            for v in sentVoc:\n",
    "                if(v.lower() not in STOP_WORDS):\n",
    "                    sentVoc_wo_stop_word.append(v)\n",
    "            sentVoc = sentVoc_wo_stop_word\n",
    "        \n",
    "            old_word = random.choice(sentVoc)\n",
    "            new_word = old_word + \" \" + old_word \n",
    "          \n",
    "            flag = 0\n",
    "            while (flag == 0):\n",
    "                if(new_word not in sentVoc):\n",
    "                    new_sents.append(sentence.replace(old_word, new_word, 1))\n",
    "                    flag = 1\n",
    "                else:\n",
    "                    new_word = random.choice(vocab_)\n",
    "    \n",
    "        new_sents_lol.append(new_sents)\n",
    "    return new_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fd9bd41-b357-414f-a555-baa250f3a15e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_map4_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on insertion of random content word\n",
    "    print(\"Generating sentences for MAP4\")\n",
    "    puncts = list(set(string.punctuation))\n",
    "    new_sents_lol = []\n",
    "    vocab_ = vocab_wo_stopw\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        for sentence in df['translation']:\n",
    "            sentVoc = [str(i) for i in tokenizer(sentence)]\n",
    "            sentVoc_wo_punct = []\n",
    "\n",
    "            for v in sentVoc:\n",
    "                if(v not in puncts):\n",
    "                    sentVoc_wo_punct.append(v)\n",
    "            sentVoc = sentVoc_wo_punct\n",
    "\n",
    "            sentVoc_wo_stop_word = []\n",
    "            for v in sentVoc:\n",
    "                if(v.lower() not in STOP_WORDS):\n",
    "                    sentVoc_wo_stop_word.append(v)\n",
    "            sentVoc = sentVoc_wo_stop_word\n",
    "        \n",
    "            new_word = random.choice(vocab_)\n",
    "            old_word = random.choice(sentVoc)\n",
    "            new_word = old_word + \" \" + new_word \n",
    "          \n",
    "            flag = 0\n",
    "            while (flag == 0):\n",
    "                if(new_word not in sentVoc):\n",
    "                    new_sents.append(sentence.replace(old_word, new_word, 1))\n",
    "                    flag = 1\n",
    "                else:\n",
    "                    new_word = random.choice(vocab_)\n",
    "    \n",
    "        new_sents_lol.append(new_sents)\n",
    "    return new_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75b6cc2b-8341-490f-8e69-b92bd6a92e41",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_map5_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on random replacements of words\n",
    "    print(\"Generating sentences for MAP5\")\n",
    "    puncts = list(set(string.punctuation))\n",
    "    new_sents_lol = []\n",
    "    vocab_ = vocab\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        for sentence in df['translation']:\n",
    "            sentVoc = [str(i) for i in tokenizer(sentence)]\n",
    "            sentVoc_wo_punct = []\n",
    "\n",
    "            for v in sentVoc:\n",
    "                if(v not in puncts):\n",
    "                    sentVoc_wo_punct.append(v)\n",
    "            sentVoc = sentVoc_wo_punct\n",
    "\n",
    "            sentVoc_wo_stop_word = []\n",
    "            for v in sentVoc:\n",
    "                if(v.lower() not in STOP_WORDS):\n",
    "                    sentVoc_wo_stop_word.append(v)\n",
    "            sentVoc = sentVoc_wo_stop_word\n",
    "            \n",
    "            new_word = random.choice(vocab_)\n",
    "            old_word = random.choice(sentVoc)\n",
    "          \n",
    "            flag = 0\n",
    "            while (flag == 0):\n",
    "                if(new_word not in sentVoc):\n",
    "                    new_sents.append(sentence.replace(old_word, new_word, 1))\n",
    "                    flag = 1\n",
    "                else:\n",
    "                    new_word = random.choice(vocab_)\n",
    "    \n",
    "        new_sents_lol.append(new_sents)\n",
    "    return new_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b5b72ff-227f-4504-b115-8ad03a9eb35b",
   "metadata": {},
   "outputs": [],
   "source": [
    "## For NLP Augmentation Library ##\n",
    "# import nlpaug.augmenter.char as nac\n",
    "import nlpaug.augmenter.word as naw\n",
    "# import nlpaug.augmenter.sentence as nas\n",
    "import nlpaug.flow as nafc\n",
    "from nlpaug.util import Action\n",
    "## For NLP Augmentation Library ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3863067d-3e9e-40fe-b0cf-b41f46045fa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "## For NLP Augmentation Library ##\n",
    "\n",
    "## This takes time ##\n",
    "\n",
    "aug = naw.ContextualWordEmbsAug( model_path='bert-base-uncased', action=\"substitute\" )\n",
    "\n",
    "## For NLP Augmentation Library ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85044a7e-f74f-46f4-82c6-09ff8ee483e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "## This function uses the NLP Augmentation Library##\n",
    "\n",
    "def generate_map6_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on synonymy replacement using BERT, considered an MAP as BERT-based replacements\n",
    "    ## have been shown to replace important content words, which do change the meaning.\n",
    "    print(\"Generating sentences for MAP6 - This takes some time for all the 20 trials.\")\n",
    "    new_sents_lol = []\n",
    "    map6_orig_sents_lol = []\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        map6_orig_sents = []\n",
    "\n",
    "        for original, sentence in zip(df['original'], df['translation']):    \n",
    "            origSent = sentence.split()\n",
    "            augmented_text = aug.augment(sentence)\n",
    "\n",
    "            if((' '.join(origSent)) != augmented_text):\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(augmented_text)\n",
    "                new_sents.append(tempL)\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(' '.join(origSent))\n",
    "                map6_orig_sents.append(tempL)\n",
    "         \n",
    "        new_sents_lol.append(new_sents)\n",
    "        map6_orig_sents_lol.append(map6_orig_sents)\n",
    "    \n",
    "    return new_sents_lol, map6_orig_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e140ebd-00b4-4ec9-a83d-f11deeb9fc4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "augAntonym = naw.AntonymAug()\n",
    "\n",
    "def generate_map7_sents(df, nTrials, stopwords=True):\n",
    "    ## returns the list of perturbed translations where words are replace by their Antonyms\n",
    "    print(\"Generating sentences for MAP7\")\n",
    "    new_sents_lol = [] \n",
    "    map7_orig_sents_lol = []\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        map7_orig_sents = []\n",
    "\n",
    "        for original, sentence in zip(df['original'], df['translation']):     \n",
    "            origSent = sentence.split()\n",
    "            augmented_text = augAntonym.augment(sentence)\n",
    "            \n",
    "            if((' '.join(origSent)) != augmented_text):  \n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(augmented_text)\n",
    "                new_sents.append(tempL)\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(' '.join(origSent))\n",
    "                map7_orig_sents.append(tempL)\n",
    "        new_sents_lol.append(new_sents)\n",
    "        map7_orig_sents_lol.append(map7_orig_sents)\n",
    "\n",
    "    return new_sents_lol, map7_orig_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6a22ad7-e2a9-4419-9f5b-88cfe39942f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_map8_sents(df, nTrials):\n",
    "    # returns a list of perturbed translations where it copyies source sentence instead of target\n",
    "    print(\"Generating sentences for MAP8\")\n",
    "    new_sents_lol = [] \n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        for sentence in df['original']:      \n",
    "            new_sents.append(sentence)\n",
    "        new_sents_lol.append(new_sents)\n",
    "\n",
    "    return new_sents_lol"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f283db28-2a50-42b5-aa0f-7d0bb0f199f2",
   "metadata": {},
   "source": [
    "### Generating sentences for MAPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9238648-bc21-4914-a8a1-2fa92d0c6822",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_sents_lol_map1, map1_orig_sents = generate_map1_sents(df_trans_thres, 20) ## Perturbations for removing negation markers\n",
    "new_sents_lol_map2  = generate_map2_sents(df_trans_thres, 20) ## Perturbations for removing random content words\n",
    "new_sents_lol_map3  = generate_map3_sents(df_trans_thres, 20) ## Perturbations for duplicating random content words\n",
    "new_sents_lol_map4  = generate_map4_sents(df_trans_thres, 20) ## Perturbations for inserting random content words\n",
    "new_sents_lol_map5  = generate_map5_sents(df_trans_thres, 20) ## Perturbations for replacing random content words\n",
    "\n",
    "## For MAP6, each trial takes ~1 minute over 16 core machine. This library uses CPU cores, and you are passing 20 trials, so this takes time.\n",
    "\n",
    "new_sents_lol_map6, map6_orig_sents = generate_map6_sents(df_trans_thres, 20) ## Perturbations for replacing target by source\n",
    "\n",
    "## For MAP6, each trial takes ~1 minute over 16 core machine. This library uses CPU cores, and you are passing 20 trials, so this takes time.\n",
    "\n",
    "new_sents_lol_map7, map7_orig_sents = generate_map7_sents(df_trans_thres, 20) ## Perturbationeachs for replacing words by their Antonyms\n",
    "new_sents_lol_map8  = generate_map8_sents(df_trans_thres, 20) ## Perturbations for replacing translation with source sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6135ba5d-d2d1-4d74-beb0-13aa7e47a28f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_mpp1_sents(df, nTrials):\n",
    "    ##returns the list of perturbed translations based removal of punctuations\n",
    "    print(\"Generating sentences for MPP1\")\n",
    "    exclude = set(string.punctuation)\n",
    "    new_sents_lol = [] \n",
    "    mpp1_orig_sents_lol = []\n",
    "    \n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        mpp1_orig_sents = []\n",
    "\n",
    "        for original, sentence in zip(df['original'], df['translation']):\n",
    "            newSentence = ''.join(ch for ch in sentence if ch not in exclude)\n",
    "            sentVoc = newSentence.split()\n",
    "            origSent = sentence.split()\n",
    "      \n",
    "            if((' '.join(origSent)) != (' '.join(sentVoc))):\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(' '.join(sentVoc))\n",
    "                new_sents.append(tempL)\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(sentence)\n",
    "                mpp1_orig_sents.append(tempL)\n",
    "        \n",
    "        new_sents_lol.append(new_sents)\n",
    "        mpp1_orig_sents_lol.append(mpp1_orig_sents)\n",
    "    \n",
    "    return new_sents_lol, mpp1_orig_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97d9ff1f-531a-453f-9f61-272220d84e92",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_mpp2_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on replacement of punctuations\n",
    "    print(\"Generating sentences for MPP2\")\n",
    "    puncts = list(set(string.punctuation))\n",
    "    new_sents_lol = [] \n",
    "    mpp2_orig_sents_lol = []\n",
    "\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        mpp2_orig_sents = []\n",
    "\n",
    "        for original, sentence in zip(df['original'], df['translation']):\n",
    "            for i in range(len(sentence)):\n",
    "                if(sentence[i] in puncts): #Found punctuation, replacing here.\n",
    "                    new_punct = random.choice(puncts)\n",
    "                    if(new_punct!=sentence[i]):\n",
    "                        newSentence = sentence.replace(sentence[i], new_punct)\n",
    "            sentVoc = newSentence.split()\n",
    "            origSent = sentence.split()\n",
    "      \n",
    "            if((' '.join(origSent)) != (' '.join(sentVoc))):\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(' '.join(sentVoc))\n",
    "                new_sents.append(tempL)\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(sentence)\n",
    "                mpp2_orig_sents.append(tempL)\n",
    "                \n",
    "        new_sents_lol.append(new_sents)\n",
    "        mpp2_orig_sents_lol.append(mpp2_orig_sents)\n",
    "    \n",
    "    return new_sents_lol, mpp2_orig_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f718226-e80f-4c10-9194-fb511adad0c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Required for MPP3/4 ##\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "## Required for MPP3/4 ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6336250-4dc4-4ba1-8df6-f6a47979590b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_mpp3_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on removal of determiners\n",
    "    print(\"Generating sentences for MPP3\")\n",
    "    puncts = list(set(string.punctuation))\n",
    "    new_sents_lol = []\n",
    "    mpp3_orig_sents_lol = []\n",
    "    \n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        mpp3_orig_sents = []\n",
    "\n",
    "        for original, sentence in zip(df['original'], df['translation']):\n",
    "            doc = nlp(sentence)\n",
    "            sentVoc = [token.text for token in doc if token.pos_ != \"DET\"]\n",
    "            origSent = sentence.split()\n",
    "            indices = []\n",
    "            for i in range(len(sentVoc)):\n",
    "                if(sentVoc[i] in puncts):\n",
    "                    sentVoc[i-1] = sentVoc[i-1]+sentVoc[i]\n",
    "                    indices.append(i)\n",
    "            iter=0\n",
    "            for idx in indices:\n",
    "                sentVoc.pop(idx-iter)\n",
    "                iter+=1\n",
    "            if((' '.join(origSent)) != (' '.join(sentVoc))):\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(' '.join(sentVoc))\n",
    "                new_sents.append(tempL)\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(sentence)\n",
    "                mpp3_orig_sents.append(tempL)\n",
    "        new_sents_lol.append(new_sents)\n",
    "        mpp3_orig_sents_lol.append(mpp3_orig_sents)\n",
    "    \n",
    "    return new_sents_lol, mpp3_orig_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fca8e3f-1e21-47e5-a027-edfab286efd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_mpp4_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on replacement of determiners from a list\n",
    "    print(\"Generating sentences for MPP4\")\n",
    "    detList =  [\"the\", \"a\", \"an\", \"this\", \"that\", \"these\", \"those\", \"my\", \"your\", \"his\", \"her\", \"its\", \"our\", \"their\", \"some\", \"any\", \"enough\", \"all\", \"each\", \"every\", \"other\", \"another\", \"such\"]\n",
    "    new_sents_lol = [] \n",
    "    mpp4_orig_sents_lol = []\n",
    "\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        mpp4_orig_sents = []\n",
    "\n",
    "        for original, sentence in zip(df['original'], df['translation']):\n",
    "            doc = nlp(sentence)\n",
    "            sentVoc = []\n",
    "            origSent = sentence.split()\n",
    "            for i in range(len(doc)):\n",
    "                if(doc[i].pos_ == \"DET\"):\n",
    "                    sentVoc.append(random.choice(detList))\n",
    "                else:\n",
    "                    sentVoc.append(str(doc[i]))\n",
    "\n",
    "            if((' '.join(origSent)) != (' '.join(sentVoc))):  \n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(' '.join(sentVoc))\n",
    "                new_sents.append(tempL)\n",
    "                tempL=[]\n",
    "                tempL.append(original)\n",
    "                tempL.append(sentence)\n",
    "                mpp4_orig_sents.append(tempL)\n",
    "\n",
    "        new_sents_lol.append(new_sents)\n",
    "        mpp4_orig_sents_lol.append(mpp4_orig_sents)\n",
    "    \n",
    "    return new_sents_lol, mpp4_orig_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5439bcc-93b5-41d5-bd28-1f3b9e683093",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_mpp5_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on uppercasing words\n",
    "    print(\"Generating sentences for MPP5\")\n",
    "    new_sents_lol = []\n",
    "\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        for sentence in df['translation']:\n",
    "            sentVoc = sentence.split()\n",
    "            for i in range(len(sentVoc)):\n",
    "                wordswithUC = []\n",
    "                wordswithUC.append(str(sentVoc[i]))\n",
    "                wordswithUC.append(str(sentVoc[i].upper()))\n",
    "                sentVoc[i] = random.choice(wordswithUC)\n",
    "          \n",
    "            new_sents.append(' '.join(sentVoc))\n",
    "        new_sents_lol.append(new_sents)\n",
    "\n",
    "    return new_sents_lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b1bd355-6dd0-4b97-8a49-9002e27cb528",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_mpp6_sents(df, nTrials):\n",
    "    ## returns the list of perturbed translations based on lowercasing words\n",
    "    print(\"Generating sentences for MPP6\")\n",
    "    new_sents_lol = []\n",
    "\n",
    "    for i in range(nTrials):\n",
    "        new_sents  = []\n",
    "        for sentence in df['translation']:\n",
    "            sentVoc = sentence.split()\n",
    "            for i in range(len(sentVoc)):\n",
    "                wordswithUC = []\n",
    "                wordswithUC.append(str(sentVoc[i]))\n",
    "                wordswithUC.append(str(sentVoc[i].lower()))\n",
    "                sentVoc[i] = random.choice(wordswithUC)\n",
    "          \n",
    "            new_sents.append(' '.join(sentVoc))\n",
    "        new_sents_lol.append(new_sents)\n",
    "\n",
    "    return new_sents_lol"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e28a4bce-0ce4-4cdc-b3ec-a479620411be",
   "metadata": {},
   "source": [
    "### Generating sentences for MPPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0a5d24e-a9b3-4654-b33e-cb7b3f9e4176",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_sents_lol_mpp1, mpp1_orig_sents = generate_mpp1_sents(df_trans_thres, 20) ## Perturbations for removing punctuations\n",
    "new_sents_lol_mpp2, mpp2_orig_sents = generate_mpp2_sents(df_trans_thres, 20) ## Perturbations for replacing punctuations\n",
    "new_sents_lol_mpp3, mpp3_orig_sents = generate_mpp3_sents(df_trans_thres, 20) ## Perturbations for removing determiners\n",
    "new_sents_lol_mpp4, mpp4_orig_sents = generate_mpp4_sents(df_trans_thres, 20) ## Perturbations for replacing determiners\n",
    "new_sents_lol_mpp5 = generate_mpp5_sents(df_trans_thres, 20) ## Perturbations for random uppercasing\n",
    "new_sents_lol_mpp6 = generate_mpp6_sents(df_trans_thres, 20) ## Perturbations for random lowercasing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d2220d3-3a59-4ea5-83e8-456e3f6658d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_scores_1(df, new_sents, nTrials):\n",
    "  scores = np.zeros((nTrials + 1 , df.shape[0]))\n",
    "  scores[0] = df['trans_score'].to_numpy() # storing prediction for original translations at index 0\n",
    "  \n",
    "  for i in tqdm(range(nTrials)):\n",
    "    embed_sent1 = cos_sim_model.encode(new_sents[i], convert_to_tensor=True)\n",
    "    embed_sent2 = cos_sim_model.encode(df['original'].values.tolist(), convert_to_tensor=True)\n",
    "    cos_sim = nn.CosineSimilarity(dim=1)(embed_sent1,embed_sent2)\n",
    "    cos_sim = (cos_sim -torch.min(cos_sim))/ (torch.max(cos_sim)-torch.min(cos_sim))\n",
    "    cos_scores = cos_sim.cpu().numpy()\n",
    "\n",
    "    bert_score_metric.add_batch(predictions=new_sents[i], references=df['original'].values.tolist())\n",
    "    score = bert_score_metric.compute(model_type=bert_score_model)\n",
    "    norm_score = (score[\"f1\"] - np.min(score[\"f1\"])) / (np.max(score[\"f1\"]) - np.min(score[\"f1\"]))\n",
    "\n",
    "    compound_metric = combine_metrics(cos_scores, norm_score, corr=[1, 1])\n",
    "    df_norm = pd.DataFrame()\n",
    "    df_norm['scores'] = compound_metric\n",
    "    df_norm['scores'] = (df_norm['scores'] - df_norm['scores'].min()) / (df_norm['scores'].max() - df_norm['scores'].min())\n",
    "    predictions = df_norm['scores'].values.tolist()\n",
    "    \n",
    "    scores[1+i] = predictions\n",
    "\n",
    "  return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbb99a19-ccc1-4e7d-a586-f6325f28e765",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_scores_2(df, orig_list_lol, new_sents_lol, nTrials):\n",
    "    x = len(orig_list_lol)\n",
    "    y = max(map(len, orig_list_lol))\n",
    "    scores_orig = np.zeros([x,y])\n",
    "    sources=[]\n",
    "    trans=[]\n",
    "    for i in range(len(orig_list_lol)):\n",
    "        temp_sources=[]\n",
    "        temp_trans=[]\n",
    "        for j in range(len(orig_list_lol[i])):\n",
    "            temp_sources.append(orig_list_lol[i][j][0])\n",
    "            temp_trans.append(orig_list_lol[i][j][1])\n",
    "        sources.append(temp_sources)\n",
    "        trans.append(temp_trans)\n",
    "    for i in range(len(sources)):\n",
    "        sent_score=[]\n",
    "        for j in range(len(sources[i])):\n",
    "            counter = 0\n",
    "            for dftrans, score in zip(df['original'], df['trans_score']):\n",
    "                if(str(sources[i][j]) == str(dftrans)):\n",
    "                    sent_score.append(score)\n",
    "                    counter = 1\n",
    "                    break;\n",
    "            if(counter == 0):\n",
    "                print(\"NF: \" + trans[i][j])\n",
    "        scores_orig[i,:len(sent_score)] = sent_score\n",
    "    print(\"Done with scores for original Sentence pairs - Trial \"+str(i+1))\n",
    "    x = len(new_sents_lol)\n",
    "    y = max(map(len, new_sents_lol))\n",
    "    scores_pert = np.zeros([x,y])\n",
    "    sources=[]\n",
    "    trans=[]\n",
    "    for i in range(len(new_sents_lol)):\n",
    "        temp_sources=[]\n",
    "        temp_trans=[]\n",
    "        for j in range(len(new_sents_lol[i])):\n",
    "            temp_sources.append(new_sents_lol[i][j][0])\n",
    "            temp_trans.append(new_sents_lol[i][j][1])\n",
    "        sources.append(temp_sources)\n",
    "        trans.append(temp_trans)\n",
    "    for j in range(len(new_sents_lol)):\n",
    "        embed_sent1 = cos_sim_model.encode(trans[i], convert_to_tensor=True)\n",
    "        embed_sent2 = cos_sim_model.encode(sources[i], convert_to_tensor=True)\n",
    "        cos_sim = nn.CosineSimilarity(dim=1)(embed_sent1,embed_sent2)\n",
    "        cos_sim = (cos_sim -torch.min(cos_sim))/ (torch.max(cos_sim)-torch.min(cos_sim))\n",
    "        cos_scores = cos_sim.cpu().numpy()\n",
    "  \n",
    "        bert_score_metric.add_batch(predictions=trans[i], references=sources[i])\n",
    "        score = bert_score_metric.compute(model_type=bert_score_model)\n",
    "        norm_score = (score[\"f1\"] - np.min(score[\"f1\"])) / (np.max(score[\"f1\"]) - np.min(score[\"f1\"]))\n",
    "        p1predictions = combine_metrics(cos_scores, norm_score, corr=[1, 1])\n",
    "        df_norm = pd.DataFrame()\n",
    "        df_norm['scores'] = p1predictions\n",
    "        df_norm['scores'] = (df_norm['scores'] - df_norm['scores'].min()) / (df_norm['scores'].max() - df_norm['scores'].min())\n",
    "        p2predictions = df_norm['scores'].values.tolist()\n",
    "        scores_pert[j,:len(p2predictions)] = p2predictions\n",
    "        if(j==19):\n",
    "            print(\"Done with scores for perturbed sentence pairs - Trial \"+str(j+1))\n",
    "        \n",
    "    return scores_orig, scores_pert"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3b1afca-0a55-4a8c-a331-37bf8282b944",
   "metadata": {},
   "source": [
    "### Generating scores and dumping them in files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d90f3ad-664f-4e5e-8fe8-6cafaebaa43f",
   "metadata": {},
   "outputs": [],
   "source": [
    "## This will take some time, score computation uses the models to get scores for each sentence pair in each trial  ##\n",
    "\n",
    "scores_map1_orig, scores_map1_perts = predict_scores_2(df_trans_thres, map1_orig_sents, new_sents_lol_map1, 20)\n",
    "scores_map2 = predict_scores_1(df_trans_thres, new_sents_lol_map2, 20)\n",
    "scores_map3 = predict_scores_1(df_trans_thres, new_sents_lol_map3, 20)\n",
    "scores_map4 = predict_scores_1(df_trans_thres, new_sents_lol_map4, 20)\n",
    "scores_map5 = predict_scores_1(df_trans_thres, new_sents_lol_map5, 20)\n",
    "scores_map6_orig, scores_map6_perts = predict_scores_2(df_trans_thres, map6_orig_sents, new_sents_lol_map6, 20)\n",
    "scores_map7_orig, scores_map7_perts = predict_scores_2(df_trans_thres, map7_orig_sents, new_sents_lol_map7, 20)\n",
    "scores_map8 = predict_scores_1(df_trans_thres, new_sents_lol_map8, 20)\n",
    "\n",
    "## This will take some time, score computation uses the models to get scores for each sentence pair in each trial  ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc121e18-ceac-400a-863c-553c542ac1b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p generated_data_{source_lang}_en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73e6a4ce-b43b-4a98-8e04-6dc07d334fa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "################## MAP1 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_map1_perts)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_map1_perts_df = pd.DataFrame(list(map(list, zip(*scores_map1_perts))), columns = columnsList)\n",
    "scores_map1_perts_df.to_csv('generated_data_'+source_lang+'_en/scores_map1_perts_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "\n",
    "columnsList = []\n",
    "for i in range(len(scores_map1_orig)):\n",
    "    columnsList.append('orig'+str(i+1))\n",
    "\n",
    "scores_map1_orig_df = pd.DataFrame(list(map(list, zip(*scores_map1_orig))), columns = columnsList)\n",
    "scores_map1_orig_df.to_csv('generated_data_'+source_lang+'_en/scores_map1_orig_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MAP2 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_map2)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_map2_df = pd.DataFrame(list(map(list, zip(*scores_map2))), columns = columnsList)\n",
    "scores_map2_df.to_csv('generated_data_'+source_lang+'_en/scores_map2_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MAP3 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_map3)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_map3_df = pd.DataFrame(list(map(list, zip(*scores_map3))), columns = columnsList)\n",
    "scores_map3_df.to_csv('generated_data_'+source_lang+'_en/scores_map3_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MAP4 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_map4)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_map4_df = pd.DataFrame(list(map(list, zip(*scores_map4))), columns = columnsList)\n",
    "scores_map4_df.to_csv('generated_data_'+source_lang+'_en/scores_map4_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MAP5 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_map5)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_map5_df = pd.DataFrame(list(map(list, zip(*scores_map5))), columns = columnsList)\n",
    "scores_map5_df.to_csv('generated_data_'+source_lang+'_en/scores_map5_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MAP6 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_map6_perts)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_map6_perts_df = pd.DataFrame(list(map(list, zip(*scores_map6_perts))), columns = columnsList)\n",
    "scores_map6_perts_df.to_csv('generated_data_'+source_lang+'_en/scores_map6_perts_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "\n",
    "columnsList = []\n",
    "for i in range(len(scores_map6_orig)):\n",
    "    columnsList.append('orig'+str(i+1))\n",
    "\n",
    "scores_map6_orig_df = pd.DataFrame(list(map(list, zip(*scores_map6_orig))), columns = columnsList)\n",
    "scores_map6_orig_df.to_csv('generated_data_'+source_lang+'_en/scores_map6_orig_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MAP7 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_map7_perts)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_map7_perts_df = pd.DataFrame(list(map(list, zip(*scores_map7_perts))), columns = columnsList)\n",
    "scores_map7_perts_df.to_csv('generated_data_'+source_lang+'_en/scores_map7_perts_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "\n",
    "columnsList = []\n",
    "for i in range(len(scores_map7_orig)):\n",
    "    columnsList.append('orig'+str(i+1))\n",
    "\n",
    "scores_map7_orig_df = pd.DataFrame(list(map(list, zip(*scores_map7_orig))), columns = columnsList)\n",
    "scores_map7_orig_df.to_csv('generated_data_'+source_lang+'_en/scores_map7_orig_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MAP8 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_map8)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_map8_df = pd.DataFrame(list(map(list, zip(*scores_map8))), columns = columnsList)\n",
    "scores_map8_df.to_csv('generated_data_'+source_lang+'_en/scores_map8_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efa98edd-bc2d-40b9-b14d-73e91e6e0bad",
   "metadata": {},
   "outputs": [],
   "source": [
    "## This will take some time, score computation uses the models to get scores for each sentence pair in each trial  ##\n",
    "\n",
    "scores_mpp1_orig, scores_mpp1_perts = predict_scores_2(df_trans_thres, mpp1_orig_sents, new_sents_lol_mpp1, 20)\n",
    "scores_mpp2_orig, scores_mpp2_perts = predict_scores_2(df_trans_thres, mpp2_orig_sents, new_sents_lol_mpp2, 20)\n",
    "scores_mpp3_orig, scores_mpp3_perts = predict_scores_2(df_trans_thres, mpp3_orig_sents, new_sents_lol_mpp3, 20)\n",
    "scores_mpp4_orig, scores_mpp4_perts = predict_scores_2(df_trans_thres, mpp4_orig_sents, new_sents_lol_mpp4, 20)\n",
    "\n",
    "scores_mpp5 = predict_scores_1(df_trans_thres, new_sents_lol_mpp5, 20)\n",
    "scores_mpp6 = predict_scores_1(df_trans_thres, new_sents_lol_mpp6, 20)\n",
    "\n",
    "## This will take some time, score computation uses the models to get scores for each sentence pair in each trial  ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98613a7a-e949-4b66-a649-fc2049d11843",
   "metadata": {},
   "outputs": [],
   "source": [
    "################## MPP1 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp1_perts)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_mpp1_perts_df = pd.DataFrame(list(map(list, zip(*scores_mpp1_perts))), columns = columnsList)\n",
    "scores_mpp1_perts_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp1_perts_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp1_orig)):\n",
    "    columnsList.append('orig'+str(i+1))\n",
    "\n",
    "scores_mpp1_orig_df = pd.DataFrame(list(map(list, zip(*scores_mpp1_orig))), columns = columnsList)\n",
    "scores_mpp1_orig_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp1_orig_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MPP2 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp2_perts)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_mpp2_perts_df = pd.DataFrame(list(map(list, zip(*scores_mpp2_perts))), columns = columnsList)\n",
    "scores_mpp2_perts_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp2_perts_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp2_orig)):\n",
    "    columnsList.append('orig'+str(i+1))\n",
    "\n",
    "scores_mpp2_orig_df = pd.DataFrame(list(map(list, zip(*scores_mpp2_orig))), columns = columnsList)\n",
    "scores_mpp2_orig_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp2_orig_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MPP3 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp3_perts)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_mpp3_perts_df = pd.DataFrame(list(map(list, zip(*scores_mpp3_perts))), columns = columnsList)\n",
    "scores_mpp3_perts_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp3_perts_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp3_orig)):\n",
    "    columnsList.append('orig'+str(i+1))\n",
    "\n",
    "scores_mpp3_orig_df = pd.DataFrame(list(map(list, zip(*scores_mpp3_orig))), columns = columnsList)\n",
    "scores_mpp3_orig_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp3_orig_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MPP4 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp4_perts)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_mpp4_perts_df = pd.DataFrame(list(map(list, zip(*scores_mpp4_perts))), columns = columnsList)\n",
    "scores_mpp4_perts_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp4_perts_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp4_orig)):\n",
    "    columnsList.append('orig'+str(i+1))\n",
    "\n",
    "scores_mpp4_orig_df = pd.DataFrame(list(map(list, zip(*scores_mpp4_orig))), columns = columnsList)\n",
    "scores_mpp4_orig_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp4_orig_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MPP5 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp5)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_mpp5_df = pd.DataFrame(list(map(list, zip(*scores_mpp5))), columns = columnsList)\n",
    "scores_mpp5_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp5_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################\n",
    "\n",
    "################## MPP6 - Dumping the score into a dataframe for computation and in a CSV file for retrieving later ##################\n",
    "columnsList = []\n",
    "for i in range(len(scores_mpp6)):\n",
    "    columnsList.append('pert'+str(i+1))\n",
    "\n",
    "scores_mpp6_df = pd.DataFrame(list(map(list, zip(*scores_mpp6))), columns = columnsList)\n",
    "scores_mpp6_df.to_csv('generated_data_'+source_lang+'_en/scores_mpp6_df_'+tqmodel+'.csv', index = False, sep=';')\n",
    "######################################################################################################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b95a49f-72f1-4e16-9566-a74d0838e711",
   "metadata": {},
   "source": [
    "### Once the scores are generated for a particular language pair, use these two code blocks below to read the scores from files. \n",
    "### If redoing experiments, this can save the time required to regenerate the scores."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac6c2660-f21d-49e4-97c2-d3001525c97c",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_map1_perts_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map1_perts_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map1_perts = scores_map1_perts_df.transpose().to_numpy()\n",
    "scores_map1_orig_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map1_orig_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map1_orig = scores_map1_orig_df.transpose().to_numpy()\n",
    "\n",
    "scores_map2_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map2_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map2 = scores_map2_df.transpose().to_numpy()\n",
    "\n",
    "scores_map3_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map3_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map3 = scores_map3_df.transpose().to_numpy()\n",
    "\n",
    "scores_map3_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map3_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map3 = scores_map3_df.transpose().to_numpy()\n",
    "\n",
    "scores_map4_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map4_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map4 = scores_map4_df.transpose().to_numpy()\n",
    "\n",
    "scores_map5_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map5_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map5 = scores_map5_df.transpose().to_numpy()\n",
    "\n",
    "scores_map6_perts_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map6_perts_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map6_perts = scores_map6_perts_df.transpose().to_numpy()\n",
    "scores_map6_orig_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map6_orig_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map6_orig = scores_map6_orig_df.transpose().to_numpy()\n",
    "\n",
    "scores_map7_perts_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map7_perts_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map7_perts = scores_map7_perts_df.transpose().to_numpy()\n",
    "scores_map7_orig_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map7_orig_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map7_orig = scores_map7_orig_df.transpose().to_numpy()\n",
    "\n",
    "scores_map8_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_map8_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_map8 = scores_map8_df.transpose().to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de0887d5-0915-462f-bb04-a269117e280c",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_mpp1_perts_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp1_perts_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp1_perts = scores_mpp1_perts_df.transpose().to_numpy()\n",
    "scores_mpp1_orig_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp1_orig_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp1_orig = scores_mpp1_orig_df.transpose().to_numpy()\n",
    "\n",
    "scores_mpp2_perts_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp2_perts_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp2_perts = scores_mpp2_perts_df.transpose().to_numpy()\n",
    "scores_mpp2_orig_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp2_orig_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp2_orig = scores_mpp2_orig_df.transpose().to_numpy()\n",
    "\n",
    "scores_mpp3_perts_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp3_perts_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp3_perts = scores_mpp3_perts_df.transpose().to_numpy()\n",
    "scores_mpp3_orig_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp3_orig_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp3_orig = scores_mpp3_orig_df.transpose().to_numpy()\n",
    "\n",
    "scores_mpp4_perts_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp4_perts_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp4_perts = scores_mpp4_perts_df.transpose().to_numpy()\n",
    "scores_mpp4_orig_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp4_orig_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp4_orig = scores_mpp4_orig_df.transpose().to_numpy()\n",
    "\n",
    "scores_mpp5_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp5_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp5 = scores_mpp5_df.transpose().to_numpy()\n",
    "\n",
    "scores_mpp6_df = pd.read_csv('generated_data_'+source_lang+'_en/scores_mpp6_df_'+tqmodel+'.csv', sep=';')\n",
    "scores_mpp6 = scores_mpp6_df.transpose().to_numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ceaba79-015f-4e1c-9fd2-ce34727722d8",
   "metadata": {},
   "source": [
    "## Computing Relative Differences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8ca653c-a8f3-436f-90d7-62d75b91ecf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_mean_map1_perts = np.mean(scores_map1_perts[0:],axis=0) ## Mean of 20 trials of perturbed scores.\n",
    "scores_mean_map1_orig = np.mean(scores_map1_orig[0:],axis=0) ## Mean of 20 trails of original translation scores\n",
    "scores_relative_diff_map1 = scores_mean_map1_orig - scores_mean_map1_perts ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_map2 = np.mean(scores_map2[1:],axis=0) ## Computing means for the 20 perturbed scores.\n",
    "scores_relative_diff_map2 = scores_map2[0] - scores_mean_map2 ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_map3 = np.mean(scores_map3[1:],axis=0) ## Computing means for the 20 perturbed scores.\n",
    "scores_relative_diff_map3 = scores_map3[0] - scores_mean_map3 ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_map4 = np.mean(scores_map4[1:],axis=0) ## Computing means for the 20 perturbed scores.\n",
    "scores_relative_diff_map4 = scores_map4[0] - scores_mean_map4 ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_map5 = np.mean(scores_map5[1:],axis=0) ## Computing means for the 20 perturbed scores.\n",
    "scores_relative_diff_map5 = scores_map5[0] - scores_mean_map5 ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_map6_perts = np.mean(scores_map6_perts[0:],axis=0) ## Mean of 20 trials of perturbed scores.\n",
    "scores_mean_map6_orig = np.mean(scores_map6_orig[0:],axis=0) ## Mean of 20 trails of original translation scores\n",
    "scores_relative_diff_map6 = scores_mean_map6_orig - scores_mean_map6_perts ## Computing relative difference \n",
    "\n",
    "scores_mean_map7_perts = np.mean(scores_map7_perts[0:],axis=0) ## Mean of 20 trials of perturbed scores.\n",
    "scores_mean_map7_orig = np.mean(scores_map7_orig[0:],axis=0) ## Mean of 20 trails of original translation scores\n",
    "scores_relative_diff_map7 = scores_mean_map7_orig - scores_mean_map7_perts ## Computing relative difference \n",
    "\n",
    "scores_mean_map8 = np.mean(scores_map8[1:],axis=0) ## Computing means for the 20 perturbed scores.\n",
    "scores_relative_diff_map8 = scores_map8[0] - scores_mean_map8 ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d42e9264-6e16-429b-ae31-b086ba536acd",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_mean_mpp1_perts = np.mean(scores_mpp1_perts[0:],axis=0) ## Mean of 20 trials of perturbed scores.\n",
    "scores_mean_mpp1_orig = np.mean(scores_mpp1_orig[0:],axis=0) ## Mean of 20 trails of original translation scores\n",
    "scores_relative_diff_mpp1 = scores_mean_mpp1_orig - scores_mean_mpp1_perts ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_mpp2_perts = np.mean(scores_mpp2_perts[0:],axis=0) ## Mean of 20 trials of perturbed scores.\n",
    "scores_mean_mpp2_orig = np.mean(scores_mpp2_orig[0:],axis=0) ## Mean of 20 trails of original translation scores\n",
    "scores_relative_diff_mpp2 = scores_mean_mpp2_orig - scores_mean_mpp2_perts ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_mpp3_perts = np.mean(scores_mpp3_perts[0:],axis=0) ## Mean of 20 trials of perturbed scores.\n",
    "scores_mean_mpp3_orig = np.mean(scores_mpp3_orig[0:],axis=0) ## Mean of 20 trails of original translation scores\n",
    "scores_relative_diff_mpp3 = scores_mean_mpp3_orig - scores_mean_mpp3_perts ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_mpp4_perts = np.mean(scores_mpp4_perts[0:],axis=0) ## Mean of 20 trials of perturbed scores.\n",
    "scores_mean_mpp4_orig = np.mean(scores_mpp4_orig[0:],axis=0) ## Mean of 20 trails of original translation scores\n",
    "scores_relative_diff_mpp4 = scores_mean_mpp4_orig - scores_mean_mpp4_perts ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_mpp5 = np.mean(scores_mpp5[1:],axis=0) ## Computing means for the 20 perturbed scores.\n",
    "scores_relative_diff_mpp5 = scores_mpp5[0] - scores_mean_mpp5 ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations.\n",
    "\n",
    "scores_mean_mpp6 = np.mean(scores_mpp6[1:],axis=0) ## Computing means for the 20 perturbed scores.\n",
    "scores_relative_diff_mpp6 = scores_mpp6[0] - scores_mean_mpp6 ## Computing relative difference among the original translation scores and scores (mean) obtained after perturbations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01b17ec5-382e-4508-a109-43b494ca4063",
   "metadata": {},
   "outputs": [],
   "source": [
    "map1_mean = np.mean(scores_relative_diff_map1)\n",
    "map2_mean = np.mean(scores_relative_diff_map2)\n",
    "map3_mean = np.mean(scores_relative_diff_map3)\n",
    "map4_mean = np.mean(scores_relative_diff_map4)\n",
    "map5_mean = np.mean(scores_relative_diff_map5)\n",
    "map6_mean = np.mean(scores_relative_diff_map6)\n",
    "map7_mean = np.mean(scores_relative_diff_map7)\n",
    "map8_mean = np.mean(scores_relative_diff_map8)\n",
    "\n",
    "mpp1_mean = np.mean(scores_relative_diff_mpp1)\n",
    "mpp2_mean = np.mean(scores_relative_diff_mpp2)\n",
    "mpp3_mean = np.mean(scores_relative_diff_mpp3)\n",
    "mpp4_mean = np.mean(scores_relative_diff_mpp4)\n",
    "mpp5_mean = np.mean(scores_relative_diff_mpp5)\n",
    "mpp6_mean = np.mean(scores_relative_diff_mpp6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c06d62e7-d7f5-4dcd-bc23-605ce7d41242",
   "metadata": {},
   "outputs": [],
   "source": [
    "map1_var = np.var(scores_relative_diff_map1)\n",
    "map2_var = np.var(scores_relative_diff_map2)\n",
    "map3_var = np.var(scores_relative_diff_map3)\n",
    "map4_var = np.var(scores_relative_diff_map4)\n",
    "map5_var = np.var(scores_relative_diff_map5)\n",
    "map6_var = np.var(scores_relative_diff_map6)\n",
    "map7_var = np.var(scores_relative_diff_map7)\n",
    "map8_var = np.var(scores_relative_diff_map8)\n",
    "\n",
    "mpp1_var = np.var(scores_relative_diff_mpp1)\n",
    "mpp2_var = np.var(scores_relative_diff_mpp2)\n",
    "mpp3_var = np.var(scores_relative_diff_mpp3)\n",
    "mpp4_var = np.var(scores_relative_diff_mpp4)\n",
    "mpp5_var = np.var(scores_relative_diff_mpp5)\n",
    "mpp6_var = np.var(scores_relative_diff_mpp6)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77931688-2a24-4ea4-baa1-110530fd810c",
   "metadata": {},
   "source": [
    "## Generating graphs and dumping scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "713755c3-010d-4017-8390-a9706dfa2c32",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_mpp = np.array([mpp1_mean, mpp2_mean, mpp3_mean, mpp4_mean, mpp5_mean, mpp6_mean])\n",
    "err_mpp = np.array([mpp1_var, mpp2_var, mpp3_var, mpp4_var, mpp5_var, mpp6_var])\n",
    "rows = [\"RemovePunctuation\", \"ReplacePunctuation\", \"RemoveDeterminers\", \"ReplaceDeterminers\", \"UPPERCASE\", \"lowercase\"]\n",
    "lang = []\n",
    "for i in range(len(X_mpp)):\n",
    "    lang.append(source_lang)\n",
    "\n",
    "dfplot = pd.DataFrame(dict(Perturbations=rows, Differences=X_mpp, error=err_mpp, Language=lang))\n",
    "\n",
    "fig = px.scatter(dfplot, title=\"Plotting Relative Difference in MPPs\", y=\"Differences\", x=\"Perturbations\", error_y=\"error\", color=\"Language\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "743813e4-826c-4234-b240-488aaa0d3707",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_map = np.array([map1_mean, map2_mean, map3_mean, map4_mean, map5_mean, map6_mean, map7_mean, map8_mean])\n",
    "err_map = np.array([map1_var, map2_var, map3_var, map4_var, map5_var, map6_var, map7_var, map8_var])\n",
    "rows = [\"RemoveNegation\", \"RemoveWords\", \"DuplicateWords\", \"InsertWords\", \"ReplaceWords\", \"BERTSentenceReplace\", \"ReplaceWordsWithAntonyms\", \"SourceAsTarget\"]\n",
    "lang = []\n",
    "for i in range(len(X_map)):\n",
    "    lang.append(source_lang)\n",
    "\n",
    "dfplot = pd.DataFrame(dict(Perturbations=rows, Differences=X_map, error=err_map, Language=lang))\n",
    "\n",
    "fig = px.scatter(dfplot, title=\"Plotting Relative Difference in MAPs\", y=\"Differences\", x=\"Perturbations\", error_y=\"error\", color=\"Language\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "481927fb-7355-44db-bd9b-6fa978fef446",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_both = np.array([mpp1_mean, mpp2_mean, mpp3_mean, mpp4_mean, mpp5_mean, mpp6_mean, map1_mean, map2_mean, map3_mean, map4_mean, map5_mean, map6_mean, map7_mean, map8_mean])\n",
    "err_both = np.array([mpp1_var, mpp2_var, mpp3_var, mpp4_var, mpp5_var, mpp6_var, map1_var, map2_var, map3_var, map4_var, map5_var, map6_var, map7_var, map8_var])\n",
    "rows = [\"RemovePunctuation\", \"ReplacePunctuation\", \"RemoveDeterminers\", \"ReplaceDeterminers\", \"UPPERCASE\", \"lowercase\", \n",
    "        \"RemoveNegation\", \"RemoveWords\", \"DuplicateWords\", \"InsertWords\", \"ReplaceWords\", \"BERTSentenceReplace\", \"ReplaceWordsWithAntonyms\", \"SourceAsTarget\"]\n",
    "lang = [\"MPP\", \"MPP\", \"MPP\", \"MPP\", \"MPP\", \"MPP\", \"MAP\", \"MAP\", \"MAP\", \"MAP\", \"MAP\", \"MAP\", \"MAP\", \"MAP\"]\n",
    "    \n",
    "dfplot = pd.DataFrame(dict(Perturbations=rows, Differences=X_both, error=err_both, Language=lang))\n",
    "\n",
    "fig = px.scatter(dfplot, title=\"Plotting Relative Difference in MPPs and MAPs\", y=\"Differences\", x=\"Perturbations\", error_y=\"error\", color=\"Language\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fa0590b-d2b6-4d4b-8b12-4ee8dd86e9c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p commonResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40dfb3b2-16fc-4250-ab5e-92b0f3b9ecd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"commonResults/\"+source_lang+\"Means_\"+tqmodel, X_both)\n",
    "np.save(\"commonResults/\"+source_lang+\"Errs_\"+tqmodel, err_both)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f39e426c-4fc1-4b86-a054-9d1f0e9018ef",
   "metadata": {},
   "source": [
    "## From here onwards, you have dumped the relative differences for all the perturbations in terms of mean values and variances. \n",
    "## You should do this for all five language pairs before being able to generate system rankings via \"getPearsonandRanking.ipynb\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}