merger_training_set.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This file implements a merger for the training sets results."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Install Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas in c:\\users\\fabio.grassiotto\\research\\venv\\ml\\lib\\site-packages (2.2.0)\n",
      "Requirement already satisfied: numpy<2,>=1.23.2 in c:\\users\\fabio.grassiotto\\research\\venv\\ml\\lib\\site-packages (from pandas) (1.26.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\fabio.grassiotto\\research\\venv\\ml\\lib\\site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\fabio.grassiotto\\research\\venv\\ml\\lib\\site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\fabio.grassiotto\\research\\venv\\ml\\lib\\site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\fabio.grassiotto\\research\\venv\\ml\\lib\\site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge training dataset results (PT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of CSV file paths\n",
    "folder_path = \"translated/final/train_sets\"\n",
    "file_paths = ['train_1.csv', 'train_2.csv', 'train_3.csv', 'train_4.csv', 'train_5.csv', 'train_6.csv', 'train_7.csv']\n",
    "\n",
    "# Initialize an empty list to store the DataFrames\n",
    "dataframes = []\n",
    "\n",
    "# Read each CSV file and append to the list\n",
    "for file in file_paths:\n",
    "    df = pd.read_csv(folder_path + \"/\" + file)\n",
    "    dataframes.append(df)\n",
    "\n",
    "# Concatenate the DataFrames row-wise\n",
    "df_train = pd.concat(dataframes, ignore_index=True)\n",
    "df_train = df_train.drop(columns=['Unnamed: 0'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answerA</th>\n",
       "      <th>answerB</th>\n",
       "      <th>answerC</th>\n",
       "      <th>correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Embora tivesse realizado trabalho doméstico pa...</td>\n",
       "      <td>O que Jesse quer fazer em seguida?</td>\n",
       "      <td>Leia o serviço de casa para Skylar</td>\n",
       "      <td>Ajuda de acabamento Skylar</td>\n",
       "      <td>skip seus estudos</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Depois da escola, o Casey conheceu a amiga num...</td>\n",
       "      <td>Porque fez o Casey isto?</td>\n",
       "      <td>ter uma boa ideia do material</td>\n",
       "      <td>brincar com um amigo</td>\n",
       "      <td>Bebam um copo e vão-se embora .</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jesse chegou rapidamente à mãe e a mãe lhes di...</td>\n",
       "      <td>Como Jesse se sentiria depois?</td>\n",
       "      <td>desperdícios</td>\n",
       "      <td>que são uma criança boa</td>\n",
       "      <td>que a mãe sempre os chama</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A Robin sabia que o Kai queria que ela gostass...</td>\n",
       "      <td>Porque é que a Robin fez isto?</td>\n",
       "      <td>Pagou-lhe para dizer que gostava.</td>\n",
       "      <td>Ela nunca gostou muito do Kai ou da moda dela .</td>\n",
       "      <td>Ela sabia que o Kai queria que a Robin gostass...</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Addison dormiu bem ontem à noite depois de jog...</td>\n",
       "      <td>Porque é que a Addison fez isto?</td>\n",
       "      <td>Recupera a sua energia.</td>\n",
       "      <td>bateu um home run</td>\n",
       "      <td>- Não .</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33405</th>\n",
       "      <td>O Quinn tinha um trabalho de pesquisa para a m...</td>\n",
       "      <td>O que quer o Quinn fazer a seguir?</td>\n",
       "      <td>Espera até ao minuto do elast</td>\n",
       "      <td>Começar o projeto à meia-noite.</td>\n",
       "      <td>Tentar terminar o trabalho deles.</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33406</th>\n",
       "      <td>Ash fez alguns novos amigos no primeiro dia na...</td>\n",
       "      <td>Porque é que o Ash fez isto?</td>\n",
       "      <td>tomar uma coca com os novos amigos.</td>\n",
       "      <td>Faz HW com o seu novo amigo.</td>\n",
       "      <td>ser sociável</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33407</th>\n",
       "      <td>Carson decidiu experimentar encontros online, ...</td>\n",
       "      <td>O que é que o Carson precisa de fazer antes di...</td>\n",
       "      <td>Casem-se antes disto.</td>\n",
       "      <td>Decidir até agora antes disto</td>\n",
       "      <td>Acabar a escola antes disto.</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33408</th>\n",
       "      <td>Jesse colocou o peru na geladeira para que ele...</td>\n",
       "      <td>Como os outros se sentiriam como resultado?</td>\n",
       "      <td>com fome</td>\n",
       "      <td>sedento</td>\n",
       "      <td>Antecipação</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33409</th>\n",
       "      <td>Robin saltou descuidadamente para a cama, mas ...</td>\n",
       "      <td>Como descreveria a Robin?</td>\n",
       "      <td>atencioso</td>\n",
       "      <td>desajeitado</td>\n",
       "      <td>Com o coração partido.</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>33410 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 context  \\\n",
       "0      Embora tivesse realizado trabalho doméstico pa...   \n",
       "1      Depois da escola, o Casey conheceu a amiga num...   \n",
       "2      Jesse chegou rapidamente à mãe e a mãe lhes di...   \n",
       "3      A Robin sabia que o Kai queria que ela gostass...   \n",
       "4      Addison dormiu bem ontem à noite depois de jog...   \n",
       "...                                                  ...   \n",
       "33405  O Quinn tinha um trabalho de pesquisa para a m...   \n",
       "33406  Ash fez alguns novos amigos no primeiro dia na...   \n",
       "33407  Carson decidiu experimentar encontros online, ...   \n",
       "33408  Jesse colocou o peru na geladeira para que ele...   \n",
       "33409  Robin saltou descuidadamente para a cama, mas ...   \n",
       "\n",
       "                                                question  \\\n",
       "0                     O que Jesse quer fazer em seguida?   \n",
       "1                               Porque fez o Casey isto?   \n",
       "2                         Como Jesse se sentiria depois?   \n",
       "3                         Porque é que a Robin fez isto?   \n",
       "4                       Porque é que a Addison fez isto?   \n",
       "...                                                  ...   \n",
       "33405                 O que quer o Quinn fazer a seguir?   \n",
       "33406                       Porque é que o Ash fez isto?   \n",
       "33407  O que é que o Carson precisa de fazer antes di...   \n",
       "33408        Como os outros se sentiriam como resultado?   \n",
       "33409                          Como descreveria a Robin?   \n",
       "\n",
       "                                   answerA  \\\n",
       "0       Leia o serviço de casa para Skylar   \n",
       "1            ter uma boa ideia do material   \n",
       "2                             desperdícios   \n",
       "3        Pagou-lhe para dizer que gostava.   \n",
       "4                  Recupera a sua energia.   \n",
       "...                                    ...   \n",
       "33405        Espera até ao minuto do elast   \n",
       "33406  tomar uma coca com os novos amigos.   \n",
       "33407                Casem-se antes disto.   \n",
       "33408                             com fome   \n",
       "33409                            atencioso   \n",
       "\n",
       "                                               answerB  \\\n",
       "0                           Ajuda de acabamento Skylar   \n",
       "1                                 brincar com um amigo   \n",
       "2                              que são uma criança boa   \n",
       "3      Ela nunca gostou muito do Kai ou da moda dela .   \n",
       "4                                    bateu um home run   \n",
       "...                                                ...   \n",
       "33405                  Começar o projeto à meia-noite.   \n",
       "33406                     Faz HW com o seu novo amigo.   \n",
       "33407                    Decidir até agora antes disto   \n",
       "33408                                          sedento   \n",
       "33409                                      desajeitado   \n",
       "\n",
       "                                                 answerC correct  \n",
       "0                                      skip seus estudos       B  \n",
       "1                        Bebam um copo e vão-se embora .       A  \n",
       "2                              que a mãe sempre os chama       B  \n",
       "3      Ela sabia que o Kai queria que a Robin gostass...       C  \n",
       "4                                                - Não .       A  \n",
       "...                                                  ...     ...  \n",
       "33405                  Tentar terminar o trabalho deles.       C  \n",
       "33406                                       ser sociável       C  \n",
       "33407                       Acabar a escola antes disto.       B  \n",
       "33408                                        Antecipação       C  \n",
       "33409                             Com o coração partido.       B  \n",
       "\n",
       "[33410 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write to filesystem\n",
    "df_train.to_csv(\"translated/final/train.csv\") "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge Ranking results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of CSV file paths\n",
    "folder_path = \"rankings/train_sets\"\n",
    "file_paths = ['train_1_rankings.csv', 'train_2_rankings.csv', 'train_3_rankings.csv',\n",
    "              'train_4_rankings.csv', 'train_5_rankings.csv', 'train_6_rankings.csv', 'train_7_rankings.csv']\n",
    "\n",
    "# Initialize an empty list to store the DataFrames\n",
    "dataframes = []\n",
    "\n",
    "# Read each CSV file and append to the list\n",
    "for file in file_paths:\n",
    "    df = pd.read_csv(folder_path + \"/\" + file)\n",
    "    dataframes.append(df)\n",
    "\n",
    "# Concatenate the DataFrames row-wise\n",
    "df_train_rankings = pd.concat(dataframes, ignore_index=True)\n",
    "df_train_rankings = df_train_rankings.drop(columns=['Unnamed: 0'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MarianMT</th>\n",
       "      <th>Unicamp_T5</th>\n",
       "      <th>NLLB_1.3b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>82</td>\n",
       "      <td>90</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>75</td>\n",
       "      <td>60</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>85</td>\n",
       "      <td>90</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>80</td>\n",
       "      <td>60</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>75</td>\n",
       "      <td>60</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33405</th>\n",
       "      <td>60</td>\n",
       "      <td>40</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33406</th>\n",
       "      <td>85</td>\n",
       "      <td>80</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33407</th>\n",
       "      <td>75</td>\n",
       "      <td>80</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33408</th>\n",
       "      <td>90</td>\n",
       "      <td>70</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33409</th>\n",
       "      <td>70</td>\n",
       "      <td>20</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>33410 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       MarianMT  Unicamp_T5  NLLB_1.3b\n",
       "0            82          90         88\n",
       "1            75          60         80\n",
       "2            85          90         80\n",
       "3            80          60         90\n",
       "4            75          60         85\n",
       "...         ...         ...        ...\n",
       "33405        60          40         80\n",
       "33406        85          80         90\n",
       "33407        75          80         90\n",
       "33408        90          70         85\n",
       "33409        70          20         80\n",
       "\n",
       "[33410 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_rankings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write to filesystem\n",
    "df_train_rankings.to_csv(\"rankings/train_rankings.csv\") "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml_proj",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}