evaluator_gemba_tst.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This file implements translation evaluations using a modified GEMBA prompt system for the test dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Install Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (2.2.2)\n",
      "Requirement already satisfied: numpy>=1.26.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from pandas) (1.26.4)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: six>=1.5 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: langchain in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (0.2.3)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (2.0.30)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (3.9.5)\n",
      "Requirement already satisfied: langchain-core<0.3.0,>=0.2.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (0.2.5)\n",
      "Requirement already satisfied: langchain-text-splitters<0.3.0,>=0.2.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (0.2.1)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (0.1.77)\n",
      "Requirement already satisfied: numpy<2,>=1 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (1.26.4)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (2.7.3)\n",
      "Requirement already satisfied: requests<3,>=2 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (2.32.2)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain) (8.3.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-core<0.3.0,>=0.2.0->langchain) (1.33)\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-core<0.3.0,>=0.2.0->langchain) (23.2)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (3.10.4)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from pydantic<3,>=1->langchain) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.4 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from pydantic<3,>=1->langchain) (2.18.4)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from pydantic<3,>=1->langchain) (4.11.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from requests<3,>=2->langchain) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from requests<3,>=2->langchain) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from requests<3,>=2->langchain) (2.2.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from requests<3,>=2->langchain) (2024.6.2)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.3)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.3.0,>=0.2.0->langchain) (3.0.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: langchain-openai in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (0.1.8)\n",
      "Requirement already satisfied: langchain-core<0.3,>=0.2.2 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-openai) (0.2.5)\n",
      "Requirement already satisfied: openai<2.0.0,>=1.26.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-openai) (1.34.0)\n",
      "Requirement already satisfied: tiktoken<1,>=0.7 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-openai) (0.7.0)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-core<0.3,>=0.2.2->langchain-openai) (6.0.1)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-core<0.3,>=0.2.2->langchain-openai) (1.33)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.66 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-core<0.3,>=0.2.2->langchain-openai) (0.1.77)\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-core<0.3,>=0.2.2->langchain-openai) (23.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-core<0.3,>=0.2.2->langchain-openai) (2.7.3)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langchain-core<0.3,>=0.2.2->langchain-openai) (8.3.0)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (4.4.0)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (1.9.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (0.27.0)\n",
      "Requirement already satisfied: sniffio in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (4.66.4)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (4.11.0)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from tiktoken<1,>=0.7->langchain-openai) (2023.10.3)\n",
      "Requirement already satisfied: requests>=2.26.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from tiktoken<1,>=0.7->langchain-openai) (2.32.2)\n",
      "Requirement already satisfied: idna>=2.8 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.26.0->langchain-openai) (3.7)\n",
      "Requirement already satisfied: certifi in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.26.0->langchain-openai) (2024.6.2)\n",
      "Requirement already satisfied: httpcore==1.* in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.26.0->langchain-openai) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.26.0->langchain-openai) (0.14.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.3,>=0.2.2->langchain-openai) (3.0.0)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from langsmith<0.2.0,>=0.1.66->langchain-core<0.3,>=0.2.2->langchain-openai) (3.10.4)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from pydantic<3,>=1->langchain-core<0.3,>=0.2.2->langchain-openai) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.4 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from pydantic<3,>=1->langchain-core<0.3,>=0.2.2->langchain-openai) (2.18.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken<1,>=0.7->langchain-openai) (2.0.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/fabiogr/miniconda3/envs/ml_proj/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken<1,>=0.7->langchain-openai) (2.2.1)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install pandas\n",
    "%pip install langchain\n",
    "%pip install langchain-openai"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import ast\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Langchain\n",
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_openai import ChatOpenAI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Globals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "LLM_MODEL_NAME=\"gpt-3.5-turbo\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility function to set up the OpenAI API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_openai_key():\n",
    "    try:\n",
    "        # Open and read the entire content of the file\n",
    "        with open(\"openai-key.txt\", 'r') as file:\n",
    "            contents = file.read()\n",
    "        \n",
    "        return contents\n",
    "    \n",
    "    except FileNotFoundError:\n",
    "        print(f\"The file does not exist.\")\n",
    "        return None\n",
    "    except Exception as e:\n",
    "        # Handle other potential exceptions (e.g., permission errors)\n",
    "        print(f\"An error occurred while reading the file: {str(e)}\")\n",
    "        return None\n",
    "\n",
    "openai_key = load_openai_key()\n",
    "os.environ[\"OPENAI_API_KEY\"] = openai_key\n",
    "\n",
    "chat = ChatOpenAI(model=\"gpt-3.5-turbo\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prompt definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "system = \"\"\"\\\n",
    "You are a helpful evaluator of the quality of translations.\n",
    "Score the following translations from English to Portuguese on a continuous scale from 0 to 100, \n",
    "where a score of zero means \"no meaning preserved\" and score of one hundred means\n",
    "\"perfect meaning and grammar\".\n",
    "\"\"\"\n",
    "\n",
    "human = \"\"\"\\\n",
    "En: {source_seg}\n",
    "Pt 1: \"{target_seg1}\" \n",
    "Pt 2: \"{target_seg2}\"\n",
    "Pt 3: \"{target_seg3}\"\n",
    "Reply with only the number scores of your evaluation, in a python list:\n",
    "\"\"\"\n",
    "prompt = ChatPromptTemplate.from_messages([(\"system\", system), (\"human\", human)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initial test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "chain = prompt | chat\n",
    "response = chain.invoke(input={\"source_seg\": \"There are more things between heaven and earth\",\n",
    "                    \"target_seg1\": \"Existem mais coisas entre o céu e a terra\",\n",
    "                    \"target_seg2\": \"Existem coisas entre o céu e a terra\",\n",
    "                    \"target_seg3\": \"Existem coisas entre o paraíso e a terra\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[90, 80, 70]\n"
     ]
    }
   ],
   "source": [
    "print(response.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "155\n"
     ]
    }
   ],
   "source": [
    "print(response.response_metadata.get('token_usage').get('total_tokens'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[90, 75, 60]\n"
     ]
    }
   ],
   "source": [
    "print(response.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_list = ast.literal_eval(response.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "90"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actual_list[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Translation Datasets for comparisons (TEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tst = pd.read_csv('data/tst.csv')\n",
    "df_tst = df_tst.drop(columns=['Unnamed: 0'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answerA</th>\n",
       "      <th>answerB</th>\n",
       "      <th>answerC</th>\n",
       "      <th>correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bailey was a nice person so she called the fam...</td>\n",
       "      <td>What will happen to Others?</td>\n",
       "      <td>talk to the family</td>\n",
       "      <td>hate bailey</td>\n",
       "      <td>thank bailey</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alex found a ring on the beach and tried to fi...</td>\n",
       "      <td>What will the person who lost want to do next?</td>\n",
       "      <td>throw it in the ocean</td>\n",
       "      <td>turn it into lost and found</td>\n",
       "      <td>leave the beach</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Quinn player with remy swinging her around and...</td>\n",
       "      <td>What will Quinn want to do next?</td>\n",
       "      <td>be friendly with Remy</td>\n",
       "      <td>have the trust of Remy</td>\n",
       "      <td>Put Remy down</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Casey was popular throughout high school. Case...</td>\n",
       "      <td>How would you describe Casey?</td>\n",
       "      <td>a lazy person</td>\n",
       "      <td>casey who has a star of football player</td>\n",
       "      <td>a social person</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alex bumped the table that the coffee was plac...</td>\n",
       "      <td>What will happen to Riley?</td>\n",
       "      <td>continue to drink the coffee</td>\n",
       "      <td>spill the coffee</td>\n",
       "      <td>have to buy a refill</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2219</th>\n",
       "      <td>Bailey spent the whole weekend at Alex's house...</td>\n",
       "      <td>How would Bailey feel afterwards?</td>\n",
       "      <td>stressed</td>\n",
       "      <td>serene</td>\n",
       "      <td>agitated</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2220</th>\n",
       "      <td>Jesse ate, drank, and talked with his friends ...</td>\n",
       "      <td>What will Jesse want to do next?</td>\n",
       "      <td>go home next</td>\n",
       "      <td>order more food next</td>\n",
       "      <td>drink all he wanted</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2221</th>\n",
       "      <td>Alex watched cartoons and had a snack after sc...</td>\n",
       "      <td>Why did Alex do this?</td>\n",
       "      <td>do homework</td>\n",
       "      <td>relax</td>\n",
       "      <td>have to think about school</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2222</th>\n",
       "      <td>Quinn name the films Jordan is most proud of a...</td>\n",
       "      <td>How would you describe Quinn?</td>\n",
       "      <td>proud of Jordan's progress</td>\n",
       "      <td>thankful to be a part of Jordan's journey</td>\n",
       "      <td>a writer</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2223</th>\n",
       "      <td>Remy smelled something horrible. After searchi...</td>\n",
       "      <td>How would Remy feel afterwards?</td>\n",
       "      <td>grossed out</td>\n",
       "      <td>disgusted</td>\n",
       "      <td>ready to eat</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2224 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                context  \\\n",
       "0     bailey was a nice person so she called the fam...   \n",
       "1     Alex found a ring on the beach and tried to fi...   \n",
       "2     Quinn player with remy swinging her around and...   \n",
       "3     Casey was popular throughout high school. Case...   \n",
       "4     Alex bumped the table that the coffee was plac...   \n",
       "...                                                 ...   \n",
       "2219  Bailey spent the whole weekend at Alex's house...   \n",
       "2220  Jesse ate, drank, and talked with his friends ...   \n",
       "2221  Alex watched cartoons and had a snack after sc...   \n",
       "2222  Quinn name the films Jordan is most proud of a...   \n",
       "2223  Remy smelled something horrible. After searchi...   \n",
       "\n",
       "                                            question  \\\n",
       "0                        What will happen to Others?   \n",
       "1     What will the person who lost want to do next?   \n",
       "2                   What will Quinn want to do next?   \n",
       "3                      How would you describe Casey?   \n",
       "4                         What will happen to Riley?   \n",
       "...                                              ...   \n",
       "2219               How would Bailey feel afterwards?   \n",
       "2220                What will Jesse want to do next?   \n",
       "2221                           Why did Alex do this?   \n",
       "2222                   How would you describe Quinn?   \n",
       "2223                 How would Remy feel afterwards?   \n",
       "\n",
       "                           answerA                                    answerB  \\\n",
       "0               talk to the family                                hate bailey   \n",
       "1            throw it in the ocean                turn it into lost and found   \n",
       "2            be friendly with Remy                     have the trust of Remy   \n",
       "3                    a lazy person    casey who has a star of football player   \n",
       "4     continue to drink the coffee                           spill the coffee   \n",
       "...                            ...                                        ...   \n",
       "2219                      stressed                                     serene   \n",
       "2220                  go home next                       order more food next   \n",
       "2221                   do homework                                      relax   \n",
       "2222    proud of Jordan's progress  thankful to be a part of Jordan's journey   \n",
       "2223                   grossed out                                  disgusted   \n",
       "\n",
       "                         answerC correct  \n",
       "0                   thank bailey       C  \n",
       "1                leave the beach       B  \n",
       "2                  Put Remy down       C  \n",
       "3                a social person       C  \n",
       "4           have to buy a refill       B  \n",
       "...                          ...     ...  \n",
       "2219                    agitated       B  \n",
       "2220         drink all he wanted       A  \n",
       "2221  have to think about school       B  \n",
       "2222                    a writer       A  \n",
       "2223                ready to eat       A  \n",
       "\n",
       "[2224 rows x 6 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tst1_pt = pd.read_csv('translated/marian_mt/tst_pt.csv')\n",
    "df_tst2_pt = pd.read_csv('translated/unicamp_t5/tst_pt.csv')\n",
    "df_tst3_pt = pd.read_csv('translated/nllb_1.3b/tst_pt.csv')\n",
    "\n",
    "# clean up data\n",
    "df_tst1_pt = df_tst1_pt.drop(columns=['Unnamed: 0.1', 'Unnamed: 0'])\n",
    "df_tst2_pt = df_tst2_pt.drop(columns=['Unnamed: 0.1', 'Unnamed: 0', 'correct'])\n",
    "df_tst3_pt = df_tst3_pt.drop(columns=['Unnamed: 0.1', 'Unnamed: 0', 'correct'])\n",
    "\n",
    "# Rename columns\n",
    "df_tst1_pt = df_tst1_pt.rename(columns={'context': 'context_1', 'question': 'question_1',\n",
    "                                  'answerA': 'answerA_1', 'answerB': 'answerB_1',\n",
    "                                  'answerC': 'answerC_1'})\n",
    "df_tst2_pt = df_tst2_pt.rename(columns={'context': 'context_2', 'question': 'question_2',\n",
    "                                  'answerA': 'answerA_2', 'answerB': 'answerB_2',\n",
    "                                  'answerC': 'answerC_2'})\n",
    "df_tst3_pt = df_tst3_pt.rename(columns={'context': 'context_3', 'question': 'question_3',\n",
    "                                  'answerA': 'answerA_3', 'answerB': 'answerB_3',\n",
    "                                  'answerC': 'answerC_3'})\n",
    "# Merge the three dataframes\n",
    "df_tst_pt = pd.concat((df_tst1_pt, df_tst2_pt, df_tst3_pt), axis=1)\n",
    "df_tst_pt = df_tst_pt[['context_1', 'context_2', 'context_3',\n",
    "                 'question_1', 'question_2', 'question_3',\n",
    "                 'answerA_1', 'answerA_2', 'answerA_3',\n",
    "                 'answerB_1', 'answerB_2', 'answerB_3',\n",
    "                 'answerC_1', 'answerC_2', 'answerC_3',\n",
    "                 'correct']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context_1</th>\n",
       "      <th>context_2</th>\n",
       "      <th>context_3</th>\n",
       "      <th>question_1</th>\n",
       "      <th>question_2</th>\n",
       "      <th>question_3</th>\n",
       "      <th>answerA_1</th>\n",
       "      <th>answerA_2</th>\n",
       "      <th>answerA_3</th>\n",
       "      <th>answerB_1</th>\n",
       "      <th>answerB_2</th>\n",
       "      <th>answerB_3</th>\n",
       "      <th>answerC_1</th>\n",
       "      <th>answerC_2</th>\n",
       "      <th>answerC_3</th>\n",
       "      <th>correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O bailey era uma pessoa simpática, por isso ch...</td>\n",
       "      <td>bailey foi uma pessoa bonita, assim ela chamou...</td>\n",
       "      <td>A Bailey era uma pessoa simpática, por isso re...</td>\n",
       "      <td>O que acontecerá com os outros?</td>\n",
       "      <td>O que acontecerá com os outros?</td>\n",
       "      <td>O que acontecerá aos Outros?</td>\n",
       "      <td>Fale com a família</td>\n",
       "      <td>conversar com a família</td>\n",
       "      <td>falar com a família</td>\n",
       "      <td>Odeia o bailey</td>\n",
       "      <td>baile de ódio</td>\n",
       "      <td>Odeio Bailey .</td>\n",
       "      <td>Graças a Deus!</td>\n",
       "      <td>agradecer bailey</td>\n",
       "      <td>Obrigado Bailey .</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alex encontrou um anel na praia e tentou encon...</td>\n",
       "      <td>Alex encontrou um anel na praia e tentou encon...</td>\n",
       "      <td>Alex encontrou um anel na praia e tentou desco...</td>\n",
       "      <td>O que a pessoa que perdeu vai querer fazer a s...</td>\n",
       "      <td>O que a pessoa que perdeu quer fazer a seguir?</td>\n",
       "      <td>O que é que a pessoa que perdeu vai querer faz...</td>\n",
       "      <td>Jogue-o no oceano</td>\n",
       "      <td>jogar no oceano</td>\n",
       "      <td>Atira-o para o oceano .</td>\n",
       "      <td>transformá-lo em perdido e encontrado</td>\n",
       "      <td>a transformar em perdido e encontrado</td>\n",
       "      <td>Transforme-o em \"perdidos e encontrados\".</td>\n",
       "      <td>Sair da praia</td>\n",
       "      <td>folgue a praia</td>\n",
       "      <td>Deixe a praia</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Quinn jogador com remy balançando ela ao redor...</td>\n",
       "      <td>Quinn jogadora com remessa girando em torno de...</td>\n",
       "      <td>A Quinn tocava com o Remy a balançá-la e ela r...</td>\n",
       "      <td>O que Quinn vai querer fazer a seguir?</td>\n",
       "      <td>O que queremos Quinn?</td>\n",
       "      <td>O que quer o Quinn fazer a seguir?</td>\n",
       "      <td>Seja amigável com Remy</td>\n",
       "      <td>ser amigo de Remy</td>\n",
       "      <td>Seja simpático com o Remy.</td>\n",
       "      <td>Tenha a confiança de Remy</td>\n",
       "      <td>ter a confiança de Remy</td>\n",
       "      <td>Tenho a confiança do Remy.</td>\n",
       "      <td>Põe o Remy no chão.</td>\n",
       "      <td>Coloque Remy</td>\n",
       "      <td>Larga o Remy.</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Casey era popular em todo o ensino médio. Case...</td>\n",
       "      <td>A casey era popular durante o ensino médio, se...</td>\n",
       "      <td>O Casey era popular no liceu, era um jogador d...</td>\n",
       "      <td>Como você descreveria Casey?</td>\n",
       "      <td>Como descreveria Casey?</td>\n",
       "      <td>Como descreveria o Casey?</td>\n",
       "      <td>uma pessoa preguiçosa</td>\n",
       "      <td>uma pessoa preguiça</td>\n",
       "      <td>uma pessoa preguiçosa</td>\n",
       "      <td>Casey que tem uma estrela do jogador de futebol</td>\n",
       "      <td>casey que tem uma estrela de jogador de futebol</td>\n",
       "      <td>Casey que tem uma estrela de futebol</td>\n",
       "      <td>uma pessoa social</td>\n",
       "      <td>uma pessoa social</td>\n",
       "      <td>pessoa social</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alex bateu na mesa em que o café foi colocado.</td>\n",
       "      <td>Alex bompeu a mesa que o café foi colocado.</td>\n",
       "      <td>Alex bateu na mesa onde o café estava colocado.</td>\n",
       "      <td>O que vai acontecer com Riley?</td>\n",
       "      <td>O que acontecerá com Riley?</td>\n",
       "      <td>O que vai acontecer ao Riley?</td>\n",
       "      <td>Continue a beber o café</td>\n",
       "      <td>continuar a beber o café</td>\n",
       "      <td>Continua a beber o café.</td>\n",
       "      <td>Derramem o café</td>\n",
       "      <td>derramar o café</td>\n",
       "      <td>derramar o café</td>\n",
       "      <td>tem que comprar um refil</td>\n",
       "      <td>ter que comprar um reenchimento</td>\n",
       "      <td>Tenho de comprar um recheio.</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2219</th>\n",
       "      <td>Bailey passou o fim de semana inteiro na casa ...</td>\n",
       "      <td>Bailey permaneceu o final de semana na casa de...</td>\n",
       "      <td>A Bailey passou o fim de semana inteiro em cas...</td>\n",
       "      <td>Como Bailey se sentiria depois?</td>\n",
       "      <td>Como Bailey se sentiria depois?</td>\n",
       "      <td>Como é que a Bailey se sentiria depois?</td>\n",
       "      <td>estressada</td>\n",
       "      <td>stress</td>\n",
       "      <td>estressado</td>\n",
       "      <td>serena</td>\n",
       "      <td>sereno</td>\n",
       "      <td>sereno</td>\n",
       "      <td>agitado</td>\n",
       "      <td>agitação</td>\n",
       "      <td>agitado</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2220</th>\n",
       "      <td>Jesse comeu, bebeu e conversou com seus amigos...</td>\n",
       "      <td>Jesse come, bebe e conversa com os amigos ante...</td>\n",
       "      <td>Jesse comeu, bebeu e conversou com os amigos a...</td>\n",
       "      <td>O que Jesse vai querer fazer a seguir?</td>\n",
       "      <td>O que Jesse quer fazer em seguida?</td>\n",
       "      <td>O que é que o Jesse vai querer fazer a seguir?</td>\n",
       "      <td>ir para casa a seguir</td>\n",
       "      <td>ir para casa seguinte</td>\n",
       "      <td>Vai para casa a seguir.</td>\n",
       "      <td>pedir mais comida a seguir</td>\n",
       "      <td>án án án án án án án án án</td>\n",
       "      <td>Pede mais comida a seguir.</td>\n",
       "      <td>Beba tudo o que ele queria</td>\n",
       "      <td>beba tudo o que ele quisesse</td>\n",
       "      <td>Beba o que quisesse.</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2221</th>\n",
       "      <td>Alex assistiu desenhos animados e tomou um lan...</td>\n",
       "      <td>Alex assistiu a cartoons e fez lanche após a e...</td>\n",
       "      <td>Alex via desenhos animados e comeu um lanche d...</td>\n",
       "      <td>Porque é que o Alex fez isto?</td>\n",
       "      <td>Por que o Alex fez isso?</td>\n",
       "      <td>Porque é que a Alex fez isto?</td>\n",
       "      <td>fazer lição de casa</td>\n",
       "      <td>fazer tarefas domésticas</td>\n",
       "      <td>Fazer lições de casa</td>\n",
       "      <td>relaxar</td>\n",
       "      <td>lúpus</td>\n",
       "      <td>Relaxa .</td>\n",
       "      <td>É preciso pensar na escola</td>\n",
       "      <td>tem que pensar a escola</td>\n",
       "      <td>Tenho de pensar na escola.</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2222</th>\n",
       "      <td>Quinn nomeia os filmes que Jordan está mais or...</td>\n",
       "      <td>Quinn nomeia os filmes Jordan é mais orgulhado...</td>\n",
       "      <td>Quinn, diz-me os filmes de que o Jordan está m...</td>\n",
       "      <td>Como você descreveria Quinn?</td>\n",
       "      <td>Como descreveria Quinn?</td>\n",
       "      <td>Como descreveria o Quinn?</td>\n",
       "      <td>Orgulhosos do progresso da Jordânia</td>\n",
       "      <td>orgulho do progresso da Jordânia</td>\n",
       "      <td>orgulhoso dos progressos do Jordan</td>\n",
       "      <td>Obrigado por fazer parte da jornada de Jordan</td>\n",
       "      <td>Agradecido por fazer parte da viagem de Jordânia</td>\n",
       "      <td>grato por fazer parte da jornada do Jordan</td>\n",
       "      <td>um escritor</td>\n",
       "      <td>um escritor</td>\n",
       "      <td>Um escritor</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2223</th>\n",
       "      <td>Remy cheirava a algo horrível. Depois de procu...</td>\n",
       "      <td>Remy sentia uma coisa horrível. Depois de proc...</td>\n",
       "      <td>Depois de procurar, Remy encontra comida com v...</td>\n",
       "      <td>Como Remy se sentiria depois?</td>\n",
       "      <td>Como se sentiria depois de Remy?</td>\n",
       "      <td>Como é que o Remy se sentiria depois?</td>\n",
       "      <td>Esgotado</td>\n",
       "      <td>grose</td>\n",
       "      <td>desagradável</td>\n",
       "      <td>enojado</td>\n",
       "      <td>desgostos</td>\n",
       "      <td>desagradável</td>\n",
       "      <td>Pronto para comer</td>\n",
       "      <td>pronta para comer</td>\n",
       "      <td>pronto para comer</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2224 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              context_1  \\\n",
       "0     O bailey era uma pessoa simpática, por isso ch...   \n",
       "1     Alex encontrou um anel na praia e tentou encon...   \n",
       "2     Quinn jogador com remy balançando ela ao redor...   \n",
       "3     Casey era popular em todo o ensino médio. Case...   \n",
       "4        Alex bateu na mesa em que o café foi colocado.   \n",
       "...                                                 ...   \n",
       "2219  Bailey passou o fim de semana inteiro na casa ...   \n",
       "2220  Jesse comeu, bebeu e conversou com seus amigos...   \n",
       "2221  Alex assistiu desenhos animados e tomou um lan...   \n",
       "2222  Quinn nomeia os filmes que Jordan está mais or...   \n",
       "2223  Remy cheirava a algo horrível. Depois de procu...   \n",
       "\n",
       "                                              context_2  \\\n",
       "0     bailey foi uma pessoa bonita, assim ela chamou...   \n",
       "1     Alex encontrou um anel na praia e tentou encon...   \n",
       "2     Quinn jogadora com remessa girando em torno de...   \n",
       "3     A casey era popular durante o ensino médio, se...   \n",
       "4           Alex bompeu a mesa que o café foi colocado.   \n",
       "...                                                 ...   \n",
       "2219  Bailey permaneceu o final de semana na casa de...   \n",
       "2220  Jesse come, bebe e conversa com os amigos ante...   \n",
       "2221  Alex assistiu a cartoons e fez lanche após a e...   \n",
       "2222  Quinn nomeia os filmes Jordan é mais orgulhado...   \n",
       "2223  Remy sentia uma coisa horrível. Depois de proc...   \n",
       "\n",
       "                                              context_3  \\\n",
       "0     A Bailey era uma pessoa simpática, por isso re...   \n",
       "1     Alex encontrou um anel na praia e tentou desco...   \n",
       "2     A Quinn tocava com o Remy a balançá-la e ela r...   \n",
       "3     O Casey era popular no liceu, era um jogador d...   \n",
       "4       Alex bateu na mesa onde o café estava colocado.   \n",
       "...                                                 ...   \n",
       "2219  A Bailey passou o fim de semana inteiro em cas...   \n",
       "2220  Jesse comeu, bebeu e conversou com os amigos a...   \n",
       "2221  Alex via desenhos animados e comeu um lanche d...   \n",
       "2222  Quinn, diz-me os filmes de que o Jordan está m...   \n",
       "2223  Depois de procurar, Remy encontra comida com v...   \n",
       "\n",
       "                                             question_1  \\\n",
       "0                       O que acontecerá com os outros?   \n",
       "1     O que a pessoa que perdeu vai querer fazer a s...   \n",
       "2                O que Quinn vai querer fazer a seguir?   \n",
       "3                          Como você descreveria Casey?   \n",
       "4                        O que vai acontecer com Riley?   \n",
       "...                                                 ...   \n",
       "2219                    Como Bailey se sentiria depois?   \n",
       "2220             O que Jesse vai querer fazer a seguir?   \n",
       "2221                      Porque é que o Alex fez isto?   \n",
       "2222                       Como você descreveria Quinn?   \n",
       "2223                      Como Remy se sentiria depois?   \n",
       "\n",
       "                                          question_2  \\\n",
       "0                    O que acontecerá com os outros?   \n",
       "1     O que a pessoa que perdeu quer fazer a seguir?   \n",
       "2                              O que queremos Quinn?   \n",
       "3                            Como descreveria Casey?   \n",
       "4                        O que acontecerá com Riley?   \n",
       "...                                              ...   \n",
       "2219                 Como Bailey se sentiria depois?   \n",
       "2220              O que Jesse quer fazer em seguida?   \n",
       "2221                        Por que o Alex fez isso?   \n",
       "2222                         Como descreveria Quinn?   \n",
       "2223                Como se sentiria depois de Remy?   \n",
       "\n",
       "                                             question_3  \\\n",
       "0                          O que acontecerá aos Outros?   \n",
       "1     O que é que a pessoa que perdeu vai querer faz...   \n",
       "2                    O que quer o Quinn fazer a seguir?   \n",
       "3                             Como descreveria o Casey?   \n",
       "4                         O que vai acontecer ao Riley?   \n",
       "...                                                 ...   \n",
       "2219            Como é que a Bailey se sentiria depois?   \n",
       "2220     O que é que o Jesse vai querer fazer a seguir?   \n",
       "2221                      Porque é que a Alex fez isto?   \n",
       "2222                          Como descreveria o Quinn?   \n",
       "2223              Como é que o Remy se sentiria depois?   \n",
       "\n",
       "                                answerA_1                         answerA_2  \\\n",
       "0                      Fale com a família           conversar com a família   \n",
       "1                       Jogue-o no oceano                   jogar no oceano   \n",
       "2                  Seja amigável com Remy                 ser amigo de Remy   \n",
       "3                   uma pessoa preguiçosa               uma pessoa preguiça   \n",
       "4                 Continue a beber o café          continuar a beber o café   \n",
       "...                                   ...                               ...   \n",
       "2219                           estressada                            stress   \n",
       "2220                ir para casa a seguir             ir para casa seguinte   \n",
       "2221                  fazer lição de casa          fazer tarefas domésticas   \n",
       "2222  Orgulhosos do progresso da Jordânia  orgulho do progresso da Jordânia   \n",
       "2223                             Esgotado                             grose   \n",
       "\n",
       "                               answerA_3  \\\n",
       "0                    falar com a família   \n",
       "1                Atira-o para o oceano .   \n",
       "2             Seja simpático com o Remy.   \n",
       "3                  uma pessoa preguiçosa   \n",
       "4               Continua a beber o café.   \n",
       "...                                  ...   \n",
       "2219                          estressado   \n",
       "2220             Vai para casa a seguir.   \n",
       "2221                Fazer lições de casa   \n",
       "2222  orgulhoso dos progressos do Jordan   \n",
       "2223                        desagradável   \n",
       "\n",
       "                                            answerB_1  \\\n",
       "0                                      Odeia o bailey   \n",
       "1               transformá-lo em perdido e encontrado   \n",
       "2                           Tenha a confiança de Remy   \n",
       "3     Casey que tem uma estrela do jogador de futebol   \n",
       "4                                     Derramem o café   \n",
       "...                                               ...   \n",
       "2219                                           serena   \n",
       "2220                       pedir mais comida a seguir   \n",
       "2221                                          relaxar   \n",
       "2222    Obrigado por fazer parte da jornada de Jordan   \n",
       "2223                                          enojado   \n",
       "\n",
       "                                             answerB_2  \\\n",
       "0                                        baile de ódio   \n",
       "1                a transformar em perdido e encontrado   \n",
       "2                              ter a confiança de Remy   \n",
       "3      casey que tem uma estrela de jogador de futebol   \n",
       "4                                      derramar o café   \n",
       "...                                                ...   \n",
       "2219                                            sereno   \n",
       "2220                       án án án án án án án án án    \n",
       "2221                                             lúpus   \n",
       "2222  Agradecido por fazer parte da viagem de Jordânia   \n",
       "2223                                         desgostos   \n",
       "\n",
       "                                       answerB_3                   answerC_1  \\\n",
       "0                                 Odeio Bailey .              Graças a Deus!   \n",
       "1      Transforme-o em \"perdidos e encontrados\".               Sair da praia   \n",
       "2                     Tenho a confiança do Remy.         Põe o Remy no chão.   \n",
       "3           Casey que tem uma estrela de futebol           uma pessoa social   \n",
       "4                                derramar o café    tem que comprar um refil   \n",
       "...                                          ...                         ...   \n",
       "2219                                      sereno                     agitado   \n",
       "2220                  Pede mais comida a seguir.  Beba tudo o que ele queria   \n",
       "2221                                    Relaxa .  É preciso pensar na escola   \n",
       "2222  grato por fazer parte da jornada do Jordan                 um escritor   \n",
       "2223                                desagradável           Pronto para comer   \n",
       "\n",
       "                            answerC_2                     answerC_3 correct  \n",
       "0                    agradecer bailey             Obrigado Bailey .       C  \n",
       "1                      folgue a praia                 Deixe a praia       B  \n",
       "2                        Coloque Remy                 Larga o Remy.       C  \n",
       "3                   uma pessoa social                 pessoa social       C  \n",
       "4     ter que comprar um reenchimento  Tenho de comprar um recheio.       B  \n",
       "...                               ...                           ...     ...  \n",
       "2219                         agitação                       agitado       B  \n",
       "2220     beba tudo o que ele quisesse          Beba o que quisesse.       A  \n",
       "2221          tem que pensar a escola    Tenho de pensar na escola.       B  \n",
       "2222                      um escritor                   Um escritor       A  \n",
       "2223                pronta para comer             pronto para comer       A  \n",
       "\n",
       "[2224 rows x 16 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tst_pt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_sequences(i):\n",
    "\n",
    "        sentence_list = ['context', 'question', 'answerA', 'answerB', 'answerC']\n",
    "        english_str=\"\"\n",
    "        for sentence in sentence_list:\n",
    "                english_str += str(df_tst.loc[i, sentence])\n",
    "                english_str += \"\\n\"\n",
    "\n",
    "        portuguese_1=\"\"\n",
    "        sentence_list = ['context_1', 'question_1', 'answerA_1', 'answerB_1', 'answerC_1']\n",
    "        for sentence in sentence_list:\n",
    "                portuguese_1 += str(df_tst_pt.loc[i, sentence])\n",
    "                portuguese_1 += \"\\n\"\n",
    "\n",
    "        portuguese_2=\"\"\n",
    "        sentence_list = ['context_2', 'question_2', 'answerA_2', 'answerB_2', 'answerC_2']\n",
    "        for sentence in sentence_list:\n",
    "                portuguese_2 += str(df_tst_pt.loc[i, sentence])\n",
    "                portuguese_2 += \"\\n\"\n",
    "\n",
    "        portuguese_3=\"\"\n",
    "        sentence_list = ['context_3', 'question_3', 'answerA_3', 'answerB_3', 'answerC_3']\n",
    "        for sentence in sentence_list:\n",
    "                portuguese_3 += str(df_tst_pt.loc[i, sentence])\n",
    "                portuguese_3 += \"\\n\"\n",
    "\n",
    "        return english_str, portuguese_1, portuguese_2, portuguese_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_translations(df_src_lang, df_tgt_lang):\n",
    "\n",
    "    total_tokens = 0\n",
    "\n",
    "    df_output = df_src_lang.copy()\n",
    "    df_output.loc[:, ['context', 'question', 'answerA', 'answerB', 'answerC']] = None\n",
    "    df_rankings = pd.DataFrame(columns=['MarianMT', 'Unicamp_T5', 'NLLB_1.3b'])\n",
    "\n",
    "    for i in tqdm(range(len(df_src_lang))):\n",
    "\n",
    "        # OpenAI call\n",
    "        chain = prompt | chat\n",
    "        english, translation_pt1, translation_pt2, translation_pt3 = create_sequences(i)\n",
    "        response = chain.invoke(input={\"source_seg\": english,\n",
    "                        \"target_seg1\": translation_pt1,\n",
    "                        \"target_seg2\": translation_pt2,\n",
    "                        \"target_seg3\": translation_pt3})\n",
    "        \n",
    "        total_tokens += response.response_metadata.get('token_usage').get('total_tokens')\n",
    "\n",
    "        results = ast.literal_eval(response.content)            \n",
    "        best = results.index(max(results))\n",
    "        \n",
    "        # Add to rankings dataset\n",
    "        df_rankings.loc[len(df_rankings)] = results\n",
    "        \n",
    "        # Copy to final dataset\n",
    "        col_list = ['context', 'question', 'answerA', 'answerB', 'answerC']\n",
    "        for col in col_list:\n",
    "            col_best = col + '_' + str(best+1)\n",
    "            best_result = df_tgt_lang.loc[i, col_best]\n",
    "            sentence_idx = col_list.index(col)\n",
    "            df_output.iat[i, sentence_idx] = best_result\n",
    "\n",
    "    print(f\"Total tokens used: {total_tokens}\")\n",
    "    \n",
    "    return df_output, df_rankings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "context     bailey was a nice person so she called the fam...\n",
       "question                          What will happen to Others?\n",
       "answerA                                    talk to the family\n",
       "answerB                                           hate bailey\n",
       "answerC                                          thank bailey\n",
       "correct                                                     C\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tst.loc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2224/2224 [23:31<00:00,  1.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total tokens used: 684752\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Test base\n",
    "df_tst_final, df_tst_rankings = get_best_translations(df_tst, df_tst_pt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answerA</th>\n",
       "      <th>answerB</th>\n",
       "      <th>answerC</th>\n",
       "      <th>correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bailey foi uma pessoa bonita, assim ela chamou...</td>\n",
       "      <td>O que acontecerá com os outros?</td>\n",
       "      <td>conversar com a família</td>\n",
       "      <td>baile de ódio</td>\n",
       "      <td>agradecer bailey</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alex encontrou um anel na praia e tentou desco...</td>\n",
       "      <td>O que é que a pessoa que perdeu vai querer faz...</td>\n",
       "      <td>Atira-o para o oceano .</td>\n",
       "      <td>Transforme-o em \"perdidos e encontrados\".</td>\n",
       "      <td>Deixe a praia</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Quinn jogador com remy balançando ela ao redor...</td>\n",
       "      <td>O que Quinn vai querer fazer a seguir?</td>\n",
       "      <td>Seja amigável com Remy</td>\n",
       "      <td>Tenha a confiança de Remy</td>\n",
       "      <td>Põe o Remy no chão.</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>O Casey era popular no liceu, era um jogador d...</td>\n",
       "      <td>Como descreveria o Casey?</td>\n",
       "      <td>uma pessoa preguiçosa</td>\n",
       "      <td>Casey que tem uma estrela de futebol</td>\n",
       "      <td>pessoa social</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alex bateu na mesa onde o café estava colocado.</td>\n",
       "      <td>O que vai acontecer ao Riley?</td>\n",
       "      <td>Continua a beber o café.</td>\n",
       "      <td>derramar o café</td>\n",
       "      <td>Tenho de comprar um recheio.</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2219</th>\n",
       "      <td>Bailey passou o fim de semana inteiro na casa ...</td>\n",
       "      <td>Como Bailey se sentiria depois?</td>\n",
       "      <td>estressada</td>\n",
       "      <td>serena</td>\n",
       "      <td>agitado</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2220</th>\n",
       "      <td>Jesse comeu, bebeu e conversou com os amigos a...</td>\n",
       "      <td>O que é que o Jesse vai querer fazer a seguir?</td>\n",
       "      <td>Vai para casa a seguir.</td>\n",
       "      <td>Pede mais comida a seguir.</td>\n",
       "      <td>Beba o que quisesse.</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2221</th>\n",
       "      <td>Alex via desenhos animados e comeu um lanche d...</td>\n",
       "      <td>Porque é que a Alex fez isto?</td>\n",
       "      <td>Fazer lições de casa</td>\n",
       "      <td>Relaxa .</td>\n",
       "      <td>Tenho de pensar na escola.</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2222</th>\n",
       "      <td>Quinn, diz-me os filmes de que o Jordan está m...</td>\n",
       "      <td>Como descreveria o Quinn?</td>\n",
       "      <td>orgulhoso dos progressos do Jordan</td>\n",
       "      <td>grato por fazer parte da jornada do Jordan</td>\n",
       "      <td>Um escritor</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2223</th>\n",
       "      <td>Depois de procurar, Remy encontra comida com v...</td>\n",
       "      <td>Como é que o Remy se sentiria depois?</td>\n",
       "      <td>desagradável</td>\n",
       "      <td>desagradável</td>\n",
       "      <td>pronto para comer</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2224 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                context  \\\n",
       "0     bailey foi uma pessoa bonita, assim ela chamou...   \n",
       "1     Alex encontrou um anel na praia e tentou desco...   \n",
       "2     Quinn jogador com remy balançando ela ao redor...   \n",
       "3     O Casey era popular no liceu, era um jogador d...   \n",
       "4       Alex bateu na mesa onde o café estava colocado.   \n",
       "...                                                 ...   \n",
       "2219  Bailey passou o fim de semana inteiro na casa ...   \n",
       "2220  Jesse comeu, bebeu e conversou com os amigos a...   \n",
       "2221  Alex via desenhos animados e comeu um lanche d...   \n",
       "2222  Quinn, diz-me os filmes de que o Jordan está m...   \n",
       "2223  Depois de procurar, Remy encontra comida com v...   \n",
       "\n",
       "                                               question  \\\n",
       "0                       O que acontecerá com os outros?   \n",
       "1     O que é que a pessoa que perdeu vai querer faz...   \n",
       "2                O que Quinn vai querer fazer a seguir?   \n",
       "3                             Como descreveria o Casey?   \n",
       "4                         O que vai acontecer ao Riley?   \n",
       "...                                                 ...   \n",
       "2219                    Como Bailey se sentiria depois?   \n",
       "2220     O que é que o Jesse vai querer fazer a seguir?   \n",
       "2221                      Porque é que a Alex fez isto?   \n",
       "2222                          Como descreveria o Quinn?   \n",
       "2223              Como é que o Remy se sentiria depois?   \n",
       "\n",
       "                                 answerA  \\\n",
       "0                conversar com a família   \n",
       "1                Atira-o para o oceano .   \n",
       "2                 Seja amigável com Remy   \n",
       "3                  uma pessoa preguiçosa   \n",
       "4               Continua a beber o café.   \n",
       "...                                  ...   \n",
       "2219                          estressada   \n",
       "2220             Vai para casa a seguir.   \n",
       "2221                Fazer lições de casa   \n",
       "2222  orgulhoso dos progressos do Jordan   \n",
       "2223                        desagradável   \n",
       "\n",
       "                                         answerB  \\\n",
       "0                                  baile de ódio   \n",
       "1      Transforme-o em \"perdidos e encontrados\".   \n",
       "2                      Tenha a confiança de Remy   \n",
       "3           Casey que tem uma estrela de futebol   \n",
       "4                                derramar o café   \n",
       "...                                          ...   \n",
       "2219                                      serena   \n",
       "2220                  Pede mais comida a seguir.   \n",
       "2221                                    Relaxa .   \n",
       "2222  grato por fazer parte da jornada do Jordan   \n",
       "2223                                desagradável   \n",
       "\n",
       "                           answerC correct  \n",
       "0                 agradecer bailey       C  \n",
       "1                    Deixe a praia       B  \n",
       "2              Põe o Remy no chão.       C  \n",
       "3                    pessoa social       C  \n",
       "4     Tenho de comprar um recheio.       B  \n",
       "...                            ...     ...  \n",
       "2219                       agitado       B  \n",
       "2220          Beba o que quisesse.       A  \n",
       "2221    Tenho de pensar na escola.       B  \n",
       "2222                   Um escritor       A  \n",
       "2223             pronto para comer       A  \n",
       "\n",
       "[2224 rows x 6 columns]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tst_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MarianMT</th>\n",
       "      <th>Unicamp_T5</th>\n",
       "      <th>NLLB_1.3b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>85</td>\n",
       "      <td>90</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85</td>\n",
       "      <td>80</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>75</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>75</td>\n",
       "      <td>60</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60</td>\n",
       "      <td>70</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2219</th>\n",
       "      <td>85</td>\n",
       "      <td>75</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2220</th>\n",
       "      <td>85</td>\n",
       "      <td>70</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2221</th>\n",
       "      <td>65</td>\n",
       "      <td>50</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2222</th>\n",
       "      <td>90</td>\n",
       "      <td>85</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2223</th>\n",
       "      <td>70</td>\n",
       "      <td>60</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2224 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      MarianMT  Unicamp_T5  NLLB_1.3b\n",
       "0           85          90         80\n",
       "1           85          80         90\n",
       "2           75          60         50\n",
       "3           75          60         80\n",
       "4           60          70         80\n",
       "...        ...         ...        ...\n",
       "2219        85          75         70\n",
       "2220        85          70         90\n",
       "2221        65          50         75\n",
       "2222        90          85         95\n",
       "2223        70          60         80\n",
       "\n",
       "[2224 rows x 3 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tst_rankings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tst_final.to_csv('translated/final/tst.csv')\n",
    "df_tst_rankings.to_csv('rankings/tst_rankings.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml_proj",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}