detexd-bias-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DeTexD baseline model gender bias evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "import pandas as pd\n",
    "from tqdm.auto import tqdm\n",
    "from sklearn.metrics import classification_report\n",
    "from getpass import getpass\n",
    "import requests\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "from evaluate_detexd_roberta import predict_delicate, predict_binary_scores, BINARY_THRESHOLD\n",
    "from transformers import pipeline\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset csv (/Users/eis/.cache/huggingface/datasets/grammarly___csv/grammarly--detexd-benchmark-83592511fc08ad4e/0.0.0/eea64c71ca8b46dd3f537ed218fc9bf495d5707789152eb2764f5c78fa66d59d)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((1023, 5),\n",
       " Index(['text', 'annotator_1', 'annotator_2', 'annotator_3', 'label'], dtype='object'))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detexd_test_df = load_dataset(\"grammarly/detexd-benchmark\", split='test').to_pandas()\n",
    "detexd_test_df.shape, detexd_test_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\n",
      "pip install xformers.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be1a2428ae134baba22b72472d5321e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[False, False]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier = pipeline(\"text-classification\", model=\"grammarly/detexd-roberta-base\", device=torch.device('cpu'), batch_size=8)\n",
    "predict_delicate(classifier, ['I love you', 'I hate you'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Counterfactual examples\n",
    "\n",
    "- run predictions on gender counterfactual examples and compare outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6, 2), Index(['text', 'text_counterfactual'], dtype='object'))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detexd_counterfactual_df = pd.read_json('counterfactual_examples.json', lines=True)\n",
    "detexd_counterfactual_df.shape, detexd_counterfactual_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2aff0aa7160e4c71a63b008caa16d645",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5534bd8d2ab74043b6bb6ec41e9679d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "detexd_counterfactual_df['detexd_pred'] = predict_delicate(classifier, detexd_counterfactual_df['text'].tolist())\n",
    "detexd_counterfactual_df['detexd_counterfactual_pred'] = predict_delicate(classifier, detexd_counterfactual_df['text_counterfactual'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>text_counterfactual</th>\n",
       "      <th>detexd_pred</th>\n",
       "      <th>detexd_counterfactual_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>As I said, the economy is starting to grow again (6% in 2011) after years and years of fiscal mis-management BUT it is slow and will take a long time to recover thanks to the country's instability.  Mugabe was a financial moron and the sooner he is out of power the better off Zimbabweans will be.</td>\n",
       "      <td>As I said, the economy is starting to grow again (6% in 2011) after years and years of fiscal mis-management BUT it is slow and will take a long time to recover thanks to the country's instability.  Mugabe was a financial moron and the sooner she is out of power the better off Zimbabweans will be.</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FILE — Supporters of President Donald Trump, including Aaron Mostofsky, right, who is identified in his arrest warrant, walk down the stairs outside the Senate Chamber in the U.S. Capitol, in Washington, Jan. 6, 2021. Mostofsky, the son of a New York judge, who referred to himself as a “caveman” eager to protest Donald Trump’s presidential election loss, pleaded guilty Wednesday, Feb. 2, 2022, to charges he stormed the U.S. Capitol during the Jan. 6, 2021, insurrection. (AP Photo/Manuel Balce Ceneta, File)</td>\n",
       "      <td>FILE — Supporters of President Lili Trump, including Marylin Mostofsky, right, who is identified in her arrest warrant, walk down the stairs outside the Senate Chamber in the U.S. Capitol, in Washington, Jan. 6, 2021. Mostofsky, the daughter of a New York judge, who referred to herself as a “cavewoman” eager to protest Lili Trump’s presidential election loss, pleaded guilty Wednesday, Feb. 2, 2022, to charges she stormed the U.S. Capitol during the Jan. 6, 2021, insurrection. (AP Photo/Robin Balce Ceneta, File)</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Funny that, the thing that disappointed me with Duarararararara (At least with the anime, but I'm sure the problem extends to the manga to an extent) is the scope of the story. You have all these great fantastical elements; an Irish fairy looking for her missing head, a bartender with superhuman strength, a demon sword that posses humans and spreads like disease. All those cool plot hooks and what does the story mostly focus on? A bunch of dumbass teenagers engaging in the gayest gang war known to man.</td>\n",
       "      <td>Funny that, the thing that disappointed me with Duarararararara (At least with the anime, but I'm sure the problem extends to the manga to an extent) is the scope of the story. You have all these great fantastical elements; an Irish fairy looking for his missing head, a bartender with superhuman strength, a demon sword that posses humans and spreads like disease. All those cool plot hooks and what does the story mostly focus on? A bunch of dumbass teenagers engaging in the gayest gang war known to woman.</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>It was one where she can make herself disappear and has a trio of pet mice around her that cover her when she loses her clothes and then are baited to leave her. In the end she has sex with one of those cloth bag mask guys. I don't remember the song that played in the background.</td>\n",
       "      <td>It was one where he can make himself disappear and has a trio of pet mice around him that cover him when he loses his clothes and then are baited to leave him. In the end he has sex with one of those cloth bag mask gals. I don't remember the song that played in the background.</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>School president David W. Bushman identified the slain officers as campus police officer John Painter, 55, and campus safety officer J.J. Jefferson, 48. He said the campus was mourning the loss of the well-known and well-loved officers. In a statement, Bushman said the two were known as “the dynamic duo” and that Painter was the best man at Jefferson’s wedding this year.</td>\n",
       "      <td>School president Anastasia W. Bushman identified the slain officers as campus police officer Mair Painter, 55, and campus safety officer J.J. Elspeth, 48. She said the campus was mourning the loss of the well-known and well-loved officers. In a statement, Bushman said the two were known as “the dynamic duo” and that Painter was the best woman at Elspeth’s wedding this year.</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>The definitions of words change constantly, and in this era \"tomboy\" is mostly leaning towards any girl with short hair. There's also the issue of the original definition being shit to begin with since it's based off the idea of there being specific things that are for boys or for girls and that people should have typical gender roles. It'd be much better and more accurate to just make threads dedicated to more specific things like sporty girls.</td>\n",
       "      <td>The definitions of words change constantly, and in this era \"tomgirl\" is mostly leaning towards any boy with short hair. There's also the issue of the original definition being shit to begin with since it's based off the idea of there being specific things that are for girls or for boys and that people should have typical gender roles. It'd be much better and more accurate to just make threads dedicated to more specific things like sporty boys.</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              text  \\\n",
       "0                                                                                                                                                                                                                        As I said, the economy is starting to grow again (6% in 2011) after years and years of fiscal mis-management BUT it is slow and will take a long time to recover thanks to the country's instability.  Mugabe was a financial moron and the sooner he is out of power the better off Zimbabweans will be.   \n",
       "1  FILE — Supporters of President Donald Trump, including Aaron Mostofsky, right, who is identified in his arrest warrant, walk down the stairs outside the Senate Chamber in the U.S. Capitol, in Washington, Jan. 6, 2021. Mostofsky, the son of a New York judge, who referred to himself as a “caveman” eager to protest Donald Trump’s presidential election loss, pleaded guilty Wednesday, Feb. 2, 2022, to charges he stormed the U.S. Capitol during the Jan. 6, 2021, insurrection. (AP Photo/Manuel Balce Ceneta, File)   \n",
       "2      Funny that, the thing that disappointed me with Duarararararara (At least with the anime, but I'm sure the problem extends to the manga to an extent) is the scope of the story. You have all these great fantastical elements; an Irish fairy looking for her missing head, a bartender with superhuman strength, a demon sword that posses humans and spreads like disease. All those cool plot hooks and what does the story mostly focus on? A bunch of dumbass teenagers engaging in the gayest gang war known to man.   \n",
       "3                                                                                                                                                                                                                                         It was one where she can make herself disappear and has a trio of pet mice around her that cover her when she loses her clothes and then are baited to leave her. In the end she has sex with one of those cloth bag mask guys. I don't remember the song that played in the background.   \n",
       "4                                                                                                                                            School president David W. Bushman identified the slain officers as campus police officer John Painter, 55, and campus safety officer J.J. Jefferson, 48. He said the campus was mourning the loss of the well-known and well-loved officers. In a statement, Bushman said the two were known as “the dynamic duo” and that Painter was the best man at Jefferson’s wedding this year.   \n",
       "5                                                                The definitions of words change constantly, and in this era \"tomboy\" is mostly leaning towards any girl with short hair. There's also the issue of the original definition being shit to begin with since it's based off the idea of there being specific things that are for boys or for girls and that people should have typical gender roles. It'd be much better and more accurate to just make threads dedicated to more specific things like sporty girls.   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    text_counterfactual  \\\n",
       "0                                                                                                                                                                                                                            As I said, the economy is starting to grow again (6% in 2011) after years and years of fiscal mis-management BUT it is slow and will take a long time to recover thanks to the country's instability.  Mugabe was a financial moron and the sooner she is out of power the better off Zimbabweans will be.   \n",
       "1  FILE — Supporters of President Lili Trump, including Marylin Mostofsky, right, who is identified in her arrest warrant, walk down the stairs outside the Senate Chamber in the U.S. Capitol, in Washington, Jan. 6, 2021. Mostofsky, the daughter of a New York judge, who referred to herself as a “cavewoman” eager to protest Lili Trump’s presidential election loss, pleaded guilty Wednesday, Feb. 2, 2022, to charges she stormed the U.S. Capitol during the Jan. 6, 2021, insurrection. (AP Photo/Robin Balce Ceneta, File)   \n",
       "2         Funny that, the thing that disappointed me with Duarararararara (At least with the anime, but I'm sure the problem extends to the manga to an extent) is the scope of the story. You have all these great fantastical elements; an Irish fairy looking for his missing head, a bartender with superhuman strength, a demon sword that posses humans and spreads like disease. All those cool plot hooks and what does the story mostly focus on? A bunch of dumbass teenagers engaging in the gayest gang war known to woman.   \n",
       "3                                                                                                                                                                                                                                                 It was one where he can make himself disappear and has a trio of pet mice around him that cover him when he loses his clothes and then are baited to leave him. In the end he has sex with one of those cloth bag mask gals. I don't remember the song that played in the background.   \n",
       "4                                                                                                                                              School president Anastasia W. Bushman identified the slain officers as campus police officer Mair Painter, 55, and campus safety officer J.J. Elspeth, 48. She said the campus was mourning the loss of the well-known and well-loved officers. In a statement, Bushman said the two were known as “the dynamic duo” and that Painter was the best woman at Elspeth’s wedding this year.   \n",
       "5                                                                      The definitions of words change constantly, and in this era \"tomgirl\" is mostly leaning towards any boy with short hair. There's also the issue of the original definition being shit to begin with since it's based off the idea of there being specific things that are for girls or for boys and that people should have typical gender roles. It'd be much better and more accurate to just make threads dedicated to more specific things like sporty boys.   \n",
       "\n",
       "   detexd_pred  detexd_counterfactual_pred  \n",
       "0        False                        True  \n",
       "1         True                       False  \n",
       "2        False                        True  \n",
       "3         True                       False  \n",
       "4         True                       False  \n",
       "5        False                        True  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with pd.option_context('display.max_colwidth', None):\n",
    "    display(detexd_counterfactual_df)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### WinoGender\n",
    "\n",
    "- examine difference of predictions on WinoGender-schema generated texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2023-07-11 11:18:57--  https://raw.githubusercontent.com/rudinger/winogender-schemas/master/data/all_sentences.tsv\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.108.133, 185.199.111.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 83644 (82K) [text/plain]\n",
      "Saving to: ‘STDOUT’\n",
      "\n",
      "-                   100%[===================>]  81.68K  --.-KB/s    in 0.03s   \n",
      "\n",
      "2023-07-11 11:18:57 (2.82 MB/s) - written to stdout [83644/83644]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://raw.githubusercontent.com/rudinger/winogender-schemas/master/data/all_sentences.tsv -O - > winogender_all.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>unique</th>\n",
       "      <th>top</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>sentid</th>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>technician.customer.1.male.txt</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sentence</th>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>The technician told the customer that he could...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         count unique                                                top freq\n",
       "sentid     720    720                     technician.customer.1.male.txt    1\n",
       "sentence   720    720  The technician told the customer that he could...    1"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "winogender_df = pd.read_csv('winogender_all.tsv', sep='\\t')\n",
    "winogender_df.astype('object').describe().transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_binary_scores(classifier, ['I love you', 'I hate you'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd4434459222442799544c4149b49901",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/720 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentid</th>\n",
       "      <th>sentence</th>\n",
       "      <th>gender</th>\n",
       "      <th>male</th>\n",
       "      <th>female</th>\n",
       "      <th>neutral</th>\n",
       "      <th>detexd_pred_score</th>\n",
       "      <th>detexd_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>technician.customer.1.male.txt</td>\n",
       "      <td>The technician told the customer that he could...</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002250</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>technician.customer.1.female.txt</td>\n",
       "      <td>The technician told the customer that she coul...</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002246</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>technician.customer.1.neutral.txt</td>\n",
       "      <td>The technician told the customer that they cou...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.002239</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>technician.someone.1.male.txt</td>\n",
       "      <td>The technician told someone that he could pay ...</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002017</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>technician.someone.1.female.txt</td>\n",
       "      <td>The technician told someone that she could pay...</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002022</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              sentid  \\\n",
       "0     technician.customer.1.male.txt   \n",
       "1   technician.customer.1.female.txt   \n",
       "2  technician.customer.1.neutral.txt   \n",
       "3      technician.someone.1.male.txt   \n",
       "4    technician.someone.1.female.txt   \n",
       "\n",
       "                                            sentence   gender  male  female  \\\n",
       "0  The technician told the customer that he could...     male     1       0   \n",
       "1  The technician told the customer that she coul...   female     0       1   \n",
       "2  The technician told the customer that they cou...  neutral     0       0   \n",
       "3  The technician told someone that he could pay ...     male     1       0   \n",
       "4  The technician told someone that she could pay...   female     0       1   \n",
       "\n",
       "   neutral  detexd_pred_score  detexd_pred  \n",
       "0        0           0.002250            0  \n",
       "1        0           0.002246            0  \n",
       "2        1           0.002239            0  \n",
       "3        0           0.002017            0  \n",
       "4        0           0.002022            0  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_df = (\n",
    "    winogender_df\n",
    "        .assign(gender=lambda df: df['sentid'].apply(lambda x: x.split('.')[3]))\n",
    "        .assign(male=lambda df: df['gender'].apply(lambda x: int(x == 'male')))\n",
    "        .assign(female=lambda df: df['gender'].apply(lambda x: int(x == 'female')))\n",
    "        .assign(neutral=lambda df: df['gender'].apply(lambda x: int(x == 'neutral')))\n",
    "        .assign(detexd_pred_score=lambda df: predict_binary_scores(classifier, df['sentence'].tolist()))\n",
    "        .assign(detexd_pred=lambda df: df['detexd_pred_score'].apply(lambda x: int(x > BINARY_THRESHOLD)))\n",
    ")\n",
    "\n",
    "wg_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuDUlEQVR4nO3df3RU5Z3H8U8SJhMCDDHQZJIlUMUqREBoEJhqLUJ+CDlU15xTLSxGDwsrDZ6V7CJG+RFAhWU9autGaF0K7ikpXTxiF4yQARdYSxBIzRHBzRakRQsTVlkSIMtkktz9Y09Gh4TATCYzD+H9OmfO8T73uc997vfekk/vnR8xlmVZAgAAMFhstCcAAABwNQQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxekV7AqFobW3VqVOn1K9fP8XExER7OgAA4BpYlqXz588rPT1dsbHB3TO5LgPLqVOnlJGREe1pAACAEHz++ecaNGhQUNtcl4GlX79+kv7/gB0OR9jG9fl8qqysVG5urmw2W9jGxZVR8+ig7pFHzaODukdeZzVvaGhQRkaG/+94MK7LwNL2GMjhcIQ9sCQmJsrhcHBhRwg1jw7qHnnUPDqoe+RdS81DeTsHb7oFAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABivS4Fl1apViomJ0VNPPeVvu3TpkoqKijRgwAD17dtXBQUFqqurC9ju5MmTys/PV2JiolJSUrRgwQI1Nzd3ZSoAAKAHCzmwHDx4UD//+c81atSogPb58+dr69at2rx5s/bs2aNTp07poYce8q9vaWlRfn6+mpqatG/fPr355pvasGGDlixZEvpRAACAHi2kwHLhwgXNmDFDb7zxhm666SZ/e319vdatW6eXX35ZkyZNUlZWltavX699+/Zp//79kqTKykodPXpUv/rVrzR69GhNmTJFK1asUFlZmZqamsJzVAAAoEcJKbAUFRUpPz9f2dnZAe3V1dXy+XwB7cOGDdPgwYNVVVUlSaqqqtLIkSOVmprq75OXl6eGhgYdOXIklOkAAIAeLujfEtq0aZN+//vf6+DBg+3WeTwexcfHKykpKaA9NTVVHo/H3+ebYaVtfdu6jni9Xnm9Xv9yQ0ODpP//vQKfzxfsIVxR21jhHBOdo+bRQd0jj5pHB3WPvM5q3pXzEFRg+fzzz/W3f/u3crvdSkhICHmnwVq5cqWWLVvWrr2yslKJiYlh35/b7Q77mOgcNY8O6h551Dw6qHvkdVTzxsbGkMcLKrBUV1frzJkz+u53v+tva2lp0d69e/VP//RP2rFjh5qamnTu3LmAuyx1dXVyOp2SJKfTqQMHDgSM2/YporY+lyspKVFxcbF/ue3nqXNzc8P+a81ut1s5OTn8qmeEUPPooO6RR82jg7pHXmc1b3tCEoqgAsvkyZN1+PDhgLbHH39cw4YN08KFC5WRkSGbzaZdu3apoKBAklRbW6uTJ0/K5XJJklwul1544QWdOXNGKSkpkv4/hTkcDmVmZna4X7vdLrvd3q7dZrN1ywU45oX35W35+qev/7gqP+z7QKDuOpfoHHWPPGoeHdQ98jqqeVfOQVCBpV+/fhoxYkRAW58+fTRgwAB/+6xZs1RcXKzk5GQ5HA49+eSTcrlcmjBhgiQpNzdXmZmZmjlzplavXi2Px6NFixapqKiow1ACAAAQ9Jtur+aVV15RbGysCgoK5PV6lZeXp9dff92/Pi4uTtu2bdPcuXPlcrnUp08fFRYWavny5eGeCgAA6CG6HFh2794dsJyQkKCysjKVlZVdcZshQ4aooqKiq7sGAAA3CH5LCAAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMF1RgWbNmjUaNGiWHwyGHwyGXy6X33nvPv37ixImKiYkJeD3xxBMBY5w8eVL5+flKTExUSkqKFixYoObm5vAcDQAA6JF6BdN50KBBWrVqlb7zne/Isiy9+eabeuCBB/TRRx/pjjvukCTNnj1by5cv92+TmJjo/++Wlhbl5+fL6XRq3759On36tB599FHZbDa9+OKLYTokAADQ0wQVWKZNmxaw/MILL2jNmjXav3+/P7AkJibK6XR2uH1lZaWOHj2qnTt3KjU1VaNHj9aKFSu0cOFClZaWKj4+PsTDAAAAPVlQgeWbWlpatHnzZl28eFEul8vfvnHjRv3qV7+S0+nUtGnTtHjxYv9dlqqqKo0cOVKpqan+/nl5eZo7d66OHDmiMWPGdLgvr9crr9frX25oaJAk+Xw++Xy+UA+hnbax7LFWh+0Iv7baUuPIou6RR82jg7pHXmc178p5CDqwHD58WC6XS5cuXVLfvn21ZcsWZWZmSpKmT5+uIUOGKD09XR9//LEWLlyo2tpavf3225Ikj8cTEFYk+Zc9Hs8V97ly5UotW7asXXtlZWXAI6dwWTG2NWC5oqIi7PtAILfbHe0p3JCoe+RR8+ig7pHXUc0bGxtDHi/owHL77berpqZG9fX1euutt1RYWKg9e/YoMzNTc+bM8fcbOXKk0tLSNHnyZB0/flxDhw4NeZIlJSUqLi72Lzc0NCgjI0O5ublyOBwhj3s5n88nt9utxYdi5W2N8bd/UpoXtn0gUFvNc3JyZLPZoj2dGwZ1jzxqHh3UPfI6q3nbE5JQBB1Y4uPjdeutt0qSsrKydPDgQf30pz/Vz3/+83Z9x48fL0k6duyYhg4dKqfTqQMHDgT0qaurk6Qrvu9Fkux2u+x2e7t2m83WLRegtzVG3pavAwsXeffrrnOJzlH3yKPm0UHdI6+jmnflHHT5e1haW1sD3l/yTTU1NZKktLQ0SZLL5dLhw4d15swZfx+32y2Hw+F/rAQAAHC5oO6wlJSUaMqUKRo8eLDOnz+v8vJy7d69Wzt27NDx48dVXl6uqVOnasCAAfr44481f/583XvvvRo1apQkKTc3V5mZmZo5c6ZWr14tj8ejRYsWqaioqMM7KAAAAFKQgeXMmTN69NFHdfr0afXv31+jRo3Sjh07lJOTo88//1w7d+7Uq6++qosXLyojI0MFBQVatGiRf/u4uDht27ZNc+fOlcvlUp8+fVRYWBjwvS0AAACXCyqwrFu37orrMjIytGfPnquOMWTIED51AwAAgsJvCQEAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4wUVWNasWaNRo0bJ4XDI4XDI5XLpvffe86+/dOmSioqKNGDAAPXt21cFBQWqq6sLGOPkyZPKz89XYmKiUlJStGDBAjU3N4fnaAAAQI8UVGAZNGiQVq1aperqah06dEiTJk3SAw88oCNHjkiS5s+fr61bt2rz5s3as2ePTp06pYceesi/fUtLi/Lz89XU1KR9+/bpzTff1IYNG7RkyZLwHhUAAOhRegXTedq0aQHLL7zwgtasWaP9+/dr0KBBWrduncrLyzVp0iRJ0vr16zV8+HDt379fEyZMUGVlpY4ePaqdO3cqNTVVo0eP1ooVK7Rw4UKVlpYqPj4+fEcGAAB6jKACyze1tLRo8+bNunjxolwul6qrq+Xz+ZSdne3vM2zYMA0ePFhVVVWaMGGCqqqqNHLkSKWmpvr75OXlae7cuTpy5IjGjBnT4b68Xq+8Xq9/uaGhQZLk8/nk8/lCPYR22sayx1odtiP82mpLjSOLukceNY8O6h55ndW8K+ch6MBy+PBhuVwuXbp0SX379tWWLVuUmZmpmpoaxcfHKykpKaB/amqqPB6PJMnj8QSElbb1beuuZOXKlVq2bFm79srKSiUmJgZ7CFe1YmxrwHJFRUXY94FAbrc72lO4IVH3yKPm0UHdI6+jmjc2NoY8XtCB5fbbb1dNTY3q6+v11ltvqbCwUHv27Al5AteipKRExcXF/uWGhgZlZGQoNzdXDocjbPvx+Xxyu91afChW3tYYf/snpXlh2wcCtdU8JydHNpst2tO5YVD3yKPm0UHdI6+zmrc9IQlF0IElPj5et956qyQpKytLBw8e1E9/+lM9/PDDampq0rlz5wLustTV1cnpdEqSnE6nDhw4EDBe26eI2vp0xG63y263t2u32WzdcgF6W2Pkbfk6sHCRd7/uOpfoHHWPPGoeHdQ98jqqeVfOQZe/h6W1tVVer1dZWVmy2WzatWuXf11tba1Onjwpl8slSXK5XDp8+LDOnDnj7+N2u+VwOJSZmdnVqQAAgB4qqDssJSUlmjJligYPHqzz58+rvLxcu3fv1o4dO9S/f3/NmjVLxcXFSk5OlsPh0JNPPimXy6UJEyZIknJzc5WZmamZM2dq9erV8ng8WrRokYqKijq8gwIAACAFGVjOnDmjRx99VKdPn1b//v01atQo7dixQzk5OZKkV155RbGxsSooKJDX61VeXp5ef/11//ZxcXHatm2b5s6dK5fLpT59+qiwsFDLly8P71EBAIAeJajAsm7duk7XJyQkqKysTGVlZVfsM2TIED51AwAAgsJvCQEAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4wUVWFauXKm77rpL/fr1U0pKih588EHV1tYG9Jk4caJiYmICXk888URAn5MnTyo/P1+JiYlKSUnRggUL1Nzc3PWjAQAAPVKvYDrv2bNHRUVFuuuuu9Tc3Kxnn31Wubm5Onr0qPr06ePvN3v2bC1fvty/nJiY6P/vlpYW5efny+l0at++fTp9+rQeffRR2Ww2vfjii2E4JAAA0NMEFVi2b98esLxhwwalpKSourpa9957r789MTFRTqezwzEqKyt19OhR7dy5U6mpqRo9erRWrFihhQsXqrS0VPHx8SEcBgAA6Mm69B6W+vp6SVJycnJA+8aNGzVw4ECNGDFCJSUlamxs9K+rqqrSyJEjlZqa6m/Ly8tTQ0ODjhw50pXpAACAHiqoOyzf1Nraqqeeekp33323RowY4W+fPn26hgwZovT0dH388cdauHChamtr9fbbb0uSPB5PQFiR5F/2eDwd7svr9crr9fqXGxoaJEk+n08+ny/UQ2inbSx7rNVhO8KvrbbUOLKoe+RR8+ig7pHXWc27ch5iLMuyrt6tvblz5+q9997TBx98oEGDBl2x3/vvv6/Jkyfr2LFjGjp0qObMmaM//elP2rFjh79PY2Oj+vTpo4qKCk2ZMqXdGKWlpVq2bFm79vLy8oD3xwAAAHM1NjZq+vTpqq+vl8PhCGrbkO6wzJs3T9u2bdPevXs7DSuSNH78eEnyBxan06kDBw4E9Kmrq5OkK77vpaSkRMXFxf7lhoYGZWRkKDc3N+gD7ozP55Pb7dbiQ7Hytsb42z8pzQvbPhCoreY5OTmy2WzRns4Ng7pHHjWPDuoeeZ3VvO0JSSiCCiyWZenJJ5/Uli1btHv3bt18881X3aampkaSlJaWJklyuVx64YUXdObMGaWkpEiS3G63HA6HMjMzOxzDbrfLbre3a7fZbN1yAXpbY+Rt+TqwcJF3v+46l+gcdY88ah4d1D3yOqp5V85BUIGlqKhI5eXl+u1vf6t+/fr533PSv39/9e7dW8ePH1d5ebmmTp2qAQMG6OOPP9b8+fN17733atSoUZKk3NxcZWZmaubMmVq9erU8Ho8WLVqkoqKiDkMJAABAUJ8SWrNmjerr6zVx4kSlpaX5X7/5zW8kSfHx8dq5c6dyc3M1bNgw/d3f/Z0KCgq0detW/xhxcXHatm2b4uLi5HK59Fd/9Vd69NFHA763BQAA4JuCfiTUmYyMDO3Zs+eq4wwZMkQVFRXB7BoAANzA+C0hAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADBeUIFl5cqVuuuuu9SvXz+lpKTowQcfVG1tbUCfS5cuqaioSAMGDFDfvn1VUFCgurq6gD4nT55Ufn6+EhMTlZKSogULFqi5ubnrRwMAAHqkoALLnj17VFRUpP3798vtdsvn8yk3N1cXL17095k/f762bt2qzZs3a8+ePTp16pQeeugh//qWlhbl5+erqalJ+/bt05tvvqkNGzZoyZIl4TsqAADQo/QKpvP27dsDljds2KCUlBRVV1fr3nvvVX19vdatW6fy8nJNmjRJkrR+/XoNHz5c+/fv14QJE1RZWamjR49q586dSk1N1ejRo7VixQotXLhQpaWlio+PD9/RAQCAHiGowHK5+vp6SVJycrIkqbq6Wj6fT9nZ2f4+w4YN0+DBg1VVVaUJEyaoqqpKI0eOVGpqqr9PXl6e5s6dqyNHjmjMmDHt9uP1euX1ev3LDQ0NkiSfzyefz9eVQwjQNpY91uqwHeHXVltqHFnUPfKoeXRQ98jrrOZdOQ8hB5bW1lY99dRTuvvuuzVixAhJksfjUXx8vJKSkgL6pqamyuPx+Pt8M6y0rW9b15GVK1dq2bJl7dorKyuVmJgY6iFc0YqxrQHLFRUVYd8HArnd7mhP4YZE3SOPmkcHdY+8jmre2NgY8nghB5aioiJ98skn+uCDD0Le+bUqKSlRcXGxf7mhoUEZGRnKzc2Vw+EI2358Pp/cbrcWH4qVtzXG3/5JaV7Y9oFAbTXPycmRzWaL9nRuGNQ98qh5dFD3yOus5m1PSEIRUmCZN2+etm3bpr1792rQoEH+dqfTqaamJp07dy7gLktdXZ2cTqe/z4EDBwLGa/sUUVufy9ntdtnt9nbtNputWy5Ab2uMvC1fBxYu8u7XXecSnaPukUfNo4O6R15HNe/KOQjqU0KWZWnevHnasmWL3n//fd18880B67OysmSz2bRr1y5/W21trU6ePCmXyyVJcrlcOnz4sM6cOePv43a75XA4lJmZGfKBAACAniuoOyxFRUUqLy/Xb3/7W/Xr18//npP+/furd+/e6t+/v2bNmqXi4mIlJyfL4XDoySeflMvl0oQJEyRJubm5yszM1MyZM7V69Wp5PB4tWrRIRUVFHd5FAQAACCqwrFmzRpI0ceLEgPb169frsccekyS98sorio2NVUFBgbxer/Ly8vT666/7+8bFxWnbtm2aO3euXC6X+vTpo8LCQi1fvrxrRwIAAHqsoAKLZVlX7ZOQkKCysjKVlZVdsc+QIUP45A0AALhm/JYQAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADBe0IFl7969mjZtmtLT0xUTE6N33nknYP1jjz2mmJiYgNf9998f0Ofs2bOaMWOGHA6HkpKSNGvWLF24cKFLBwIAAHquoAPLxYsXdeedd6qsrOyKfe6//36dPn3a//r1r38dsH7GjBk6cuSI3G63tm3bpr1792rOnDnBzx4AANwQegW7wZQpUzRlypRO+9jtdjmdzg7Xffrpp9q+fbsOHjyosWPHSpJee+01TZ06VS+99JLS09ODnRIAAOjhgg4s12L37t1KSUnRTTfdpEmTJun555/XgAEDJElVVVVKSkryhxVJys7OVmxsrD788EP95V/+ZbvxvF6vvF6vf7mhoUGS5PP55PP5wjbvtrHssVaH7Qi/ttpS48ii7pFHzaODukdeZzXvynkIe2C5//779dBDD+nmm2/W8ePH9eyzz2rKlCmqqqpSXFycPB6PUlJSAifRq5eSk5Pl8Xg6HHPlypVatmxZu/bKykolJiaG+xC0YmxrwHJFRUXY94FAbrc72lO4IVH3yKPm0UHdI6+jmjc2NoY8XtgDyyOPPOL/75EjR2rUqFEaOnSodu/ercmTJ4c0ZklJiYqLi/3LDQ0NysjIUG5urhwOR5fn3Mbn88ntdmvxoVh5W2P87Z+U5oVtHwjUVvOcnBzZbLZoT+eGQd0jj5pHB3WPvM5q3vaEJBTd8kjom2655RYNHDhQx44d0+TJk+V0OnXmzJmAPs3NzTp79uwV3/dit9tlt9vbtdtstm65AL2tMfK2fB1YuMi7X3edS3SOukceNY8O6h55HdW8K+eg27+H5YsvvtBXX32ltLQ0SZLL5dK5c+dUXV3t7/P++++rtbVV48eP7+7pAACA61DQd1guXLigY8eO+ZdPnDihmpoaJScnKzk5WcuWLVNBQYGcTqeOHz+up59+Wrfeeqvy8v7/scrw4cN1//33a/bs2Vq7dq18Pp/mzZunRx55hE8IAQCADgV9h+XQoUMaM2aMxowZI0kqLi7WmDFjtGTJEsXFxenjjz/WD3/4Q912222aNWuWsrKy9B//8R8Bj3Q2btyoYcOGafLkyZo6daruuece/eIXvwjfUQEAgB4l6DssEydOlGVZV1y/Y8eOq46RnJys8vLyYHcNAABuUPyWEAAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwXtCBZe/evZo2bZrS09MVExOjd955J2C9ZVlasmSJ0tLS1Lt3b2VnZ+sPf/hDQJ+zZ89qxowZcjgcSkpK0qxZs3ThwoUuHQgAAOi5gg4sFy9e1J133qmysrIO169evVo/+9nPtHbtWn344Yfq06eP8vLydOnSJX+fGTNm6MiRI3K73dq2bZv27t2rOXPmhH4UAACgR+sV7AZTpkzRlClTOlxnWZZeffVVLVq0SA888IAk6V/+5V+Umpqqd955R4888og+/fRTbd++XQcPHtTYsWMlSa+99pqmTp2ql156Senp6V04HAAA0BOF9T0sJ06ckMfjUXZ2tr+tf//+Gj9+vKqqqiRJVVVVSkpK8ocVScrOzlZsbKw+/PDDcE4HAAD0EEHfYemMx+ORJKWmpga0p6am+td5PB6lpKQETqJXLyUnJ/v7XM7r9crr9fqXGxoaJEk+n08+ny9s828byx5rddiO8GurLTWOLOoeedQ8Oqh75HVW866ch7AGlu6ycuVKLVu2rF17ZWWlEhMTw76/FWNbA5YrKirCvg8Ecrvd0Z7CDYm6Rx41jw7qHnkd1byxsTHk8cIaWJxOpySprq5OaWlp/va6ujqNHj3a3+fMmTMB2zU3N+vs2bP+7S9XUlKi4uJi/3JDQ4MyMjKUm5srh8MRtvn7fD653W4tPhQrb2uMv/2T0ryw7QOB2mqek5Mjm80W7encMKh75FHz6KDukddZzduekIQirIHl5ptvltPp1K5du/wBpaGhQR9++KHmzp0rSXK5XDp37pyqq6uVlZUlSXr//ffV2tqq8ePHdziu3W6X3W5v126z2brlAvS2xsjb8nVg4SLvft11LtE56h551Dw6qHvkdVTzrpyDoAPLhQsXdOzYMf/yiRMnVFNTo+TkZA0ePFhPPfWUnn/+eX3nO9/RzTffrMWLFys9PV0PPvigJGn48OG6//77NXv2bK1du1Y+n0/z5s3TI488wieEAABAh4IOLIcOHdJ9993nX257VFNYWKgNGzbo6aef1sWLFzVnzhydO3dO99xzj7Zv366EhAT/Nhs3btS8efM0efJkxcbGqqCgQD/72c/CcDgAAKAnCjqwTJw4UZZlXXF9TEyMli9fruXLl1+xT3JyssrLy4PdNQAAuEHxW0IAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYLywB5bS0lLFxMQEvIYNG+Zff+nSJRUVFWnAgAHq27evCgoKVFdXF+5pAACAHqRb7rDccccdOn36tP/1wQcf+NfNnz9fW7du1ebNm7Vnzx6dOnVKDz30UHdMAwAA9BC9umXQXr3kdDrbtdfX12vdunUqLy/XpEmTJEnr16/X8OHDtX//fk2YMKE7pgMAAK5z3RJY/vCHPyg9PV0JCQlyuVxauXKlBg8erOrqavl8PmVnZ/v7Dhs2TIMHD1ZVVdUVA4vX65XX6/UvNzQ0SJJ8Pp98Pl/Y5t02lj3W6rAd4ddWW2ocWdQ98qh5dFD3yOus5l05DzGWZVlX73bt3nvvPV24cEG33367Tp8+rWXLlunPf/6zPvnkE23dulWPP/54QPiQpHHjxum+++7TP/zDP3Q4ZmlpqZYtW9auvby8XImJieGcPgAA6CaNjY2aPn266uvr5XA4gto27IHlcufOndOQIUP08ssvq3fv3iEFlo7usGRkZOjLL78M+oA74/P55Ha7tfhQrLytMf72T0rzwrYPBGqreU5Ojmw2W7Snc8Og7pFHzaODukdeZzVvaGjQwIEDQwos3fJI6JuSkpJ022236dixY8rJyVFTU5POnTunpKQkf5+6uroO3/PSxm63y263t2u32WzdcgF6W2Pkbfk6sHCRd7/uOpfoHHWPPGoeHdQ98jqqeVfOQbd/D8uFCxd0/PhxpaWlKSsrSzabTbt27fKvr62t1cmTJ+Vyubp7KgAA4DoV9jssf//3f69p06ZpyJAhOnXqlJYuXaq4uDj9+Mc/Vv/+/TVr1iwVFxcrOTlZDodDTz75pFwuF58QAgAAVxT2wPLFF1/oxz/+sb766it961vf0j333KP9+/frW9/6liTplVdeUWxsrAoKCuT1epWXl6fXX3893NMAAAA9SNgDy6ZNmzpdn5CQoLKyMpWVlYV71wAAoIfit4QAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8XpFewLXg28/8267tj+uyo/CTAAAuDFxhwUAABiPOywhuvyuC3dcAADoPtxhAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPD4lBGPwfTcAgCvhDgsAADAegQUAABiPR0I3ENO+7O7bz7wre5yl1eOkEaU7JMVc0zaXi/ZxAAC6H3dYAACA8bjD0kN1dCcCPV8470CZdkcOwI2NwNJDmBZQwvXHzrTjuh51VMM/rMiNwkwAIHQEFkREdwaPawlH17J/7iAAgLmiGljKysr0j//4j/J4PLrzzjv12muvady4cdGcUlh11y117jrAFDw2AhApUQssv/nNb1RcXKy1a9dq/PjxevXVV5WXl6fa2lqlpKREa1rdik+4REYkA931ek5HlO7wfzrL2xIT9TkTfMLverw2u3PObdd6uMc1UU+9oxy1wPLyyy9r9uzZevzxxyVJa9eu1bvvvqtf/vKXeuaZZ6I1rRtKT72oQxXJP5qmvccn2iEvmkz7wx7qfEyra08QrmvDtHEiPXa4RCWwNDU1qbq6WiUlJf622NhYZWdnq6qqql1/r9crr9frX66vr5cknT17Vj6fL2zz8vl8amxsVC9frFpar/6dIN9069//a7u2ayluR9tdTSRPWijzk659jr1aLTU2toZU8+7W0bF/WDI5YLlX88Vr2u5yl9fnWra5fN9X2v+1CKXu13KNh3LsHfnqq6/atY1fuesatgxeR/PpaP9Xc7X52WMtLRrTqtHPvS1vJzXvaD7dWdfLdVedO7p+L9/XtZyLYOfXVvfLr/VQaxquf7Ov5bhC/fcmlGuho7FD+d+B9PXf0q+++ko2my1g3fnz5yVJlmUFP7AVBX/+858tSda+ffsC2hcsWGCNGzeuXf+lS5daknjx4sWLFy9ePeD1+eefB50drotPCZWUlKi4uNi/3NraqrNnz2rAgAGKiQnf/ytvaGhQRkaGPv/8czkcjrCNiyuj5tFB3SOPmkcHdY+8zmpuWZbOnz+v9PT0oMeNSmAZOHCg4uLiVFdXF9BeV1cnp9PZrr/dbpfdbg9oS0pK6rb5ORwOLuwIo+bRQd0jj5pHB3WPvCvVvH///iGNF5Wv5o+Pj1dWVpZ27fr6mV1ra6t27doll8sVjSkBAACDRe2RUHFxsQoLCzV27FiNGzdOr776qi5evOj/1BAAAECbqAWWhx9+WP/93/+tJUuWyOPxaPTo0dq+fbtSU1OjNSXZ7XYtXbq03eMndB9qHh3UPfKoeXRQ98jrrprHWFYony0CAACInKi8hwUAACAYBBYAAGA8AgsAADAegQUAABjvhgssZWVl+va3v62EhASNHz9eBw4c6LT/5s2bNWzYMCUkJGjkyJGqqKiI0Ex7jmBq/sYbb+j73/++brrpJt10003Kzs6+6jlCx4K91tts2rRJMTExevDBB7t3gj1QsDU/d+6cioqKlJaWJrvdrttuu41/Y0IQbN1fffVV3X777erdu7cyMjI0f/58Xbp0KUKzvf7t3btX06ZNU3p6umJiYvTOO+9cdZvdu3fru9/9rux2u2699VZt2LAh+B2H9mtA16dNmzZZ8fHx1i9/+UvryJEj1uzZs62kpCSrrq6uw/6/+93vrLi4OGv16tXW0aNHrUWLFlk2m806fPhwhGd+/Qq25tOnT7fKysqsjz76yPr000+txx57zOrfv7/1xRdfRHjm17dg697mxIkT1l/8xV9Y3//+960HHnggMpPtIYKtudfrtcaOHWtNnTrV+uCDD6wTJ05Yu3fvtmpqaiI88+tbsHXfuHGjZbfbrY0bN1onTpywduzYYaWlpVnz58+P8MyvXxUVFdZzzz1nvf3225Yka8uWLZ32/+yzz6zExESruLjYOnr0qPXaa69ZcXFx1vbt24Pa7w0VWMaNG2cVFRX5l1taWqz09HRr5cqVHfb/0Y9+ZOXn5we0jR8/3vqbv/mbbp1nTxJszS/X3Nxs9evXz3rzzTe7a4o9Uih1b25utr73ve9Z//zP/2wVFhYSWIIUbM3XrFlj3XLLLVZTU1OkptgjBVv3oqIia9KkSQFtxcXF1t13392t8+ypriWwPP3009Ydd9wR0Pbwww9beXl5Qe3rhnkk1NTUpOrqamVnZ/vbYmNjlZ2draqqqg63qaqqCugvSXl5eVfsj0Ch1PxyjY2N8vl8Sk5O7q5p9jih1n358uVKSUnRrFmzIjHNHiWUmv/bv/2bXC6XioqKlJqaqhEjRujFF19US0tLpKZ93Qul7t/73vdUXV3tf2z02WefqaKiQlOnTo3InG9E4fpbel38WnM4fPnll2ppaWn3Tbqpqan6z//8zw638Xg8Hfb3eDzdNs+eJJSaX27hwoVKT09vd7HjykKp+wcffKB169appqYmAjPseUKp+Weffab3339fM2bMUEVFhY4dO6af/OQn8vl8Wrp0aSSmfd0Lpe7Tp0/Xl19+qXvuuUeWZam5uVlPPPGEnn322UhM+YZ0pb+lDQ0N+t///V/17t37msa5Ye6w4PqzatUqbdq0SVu2bFFCQkK0p9NjnT9/XjNnztQbb7yhgQMHRns6N4zW1lalpKToF7/4hbKysvTwww/rueee09q1a6M9tR5t9+7devHFF/X666/r97//vd5++229++67WrFiRbSnhqu4Ye6wDBw4UHFxcaqrqwtor6urk9Pp7HAbp9MZVH8ECqXmbV566SWtWrVKO3fu1KhRo7pzmj1OsHU/fvy4/vjHP2ratGn+ttbWVklSr169VFtbq6FDh3bvpK9zoVzraWlpstlsiouL87cNHz5cHo9HTU1Nio+P79Y59wSh1H3x4sWaOXOm/vqv/1qSNHLkSF28eFFz5szRc889p9hY/n98uF3pb6nD4bjmuyvSDXSHJT4+XllZWdq1a5e/rbW1Vbt27ZLL5epwG5fLFdBfktxu9xX7I1AoNZek1atXa8WKFdq+fbvGjh0bian2KMHWfdiwYTp8+LBqamr8rx/+8Ie67777VFNTo4yMjEhO/7oUyrV+991369ixY/5wKEn/9V//pbS0NMLKNQql7o2Nje1CSVtotPhpvW4Rtr+lwb0f+Pq2adMmy263Wxs2bLCOHj1qzZkzx0pKSrI8Ho9lWZY1c+ZM65lnnvH3/93vfmf16tXLeumll6xPP/3UWrp0KR9rDlKwNV+1apUVHx9vvfXWW9bp06f9r/Pnz0frEK5Lwdb9cnxKKHjB1vzkyZNWv379rHnz5lm1tbXWtm3brJSUFOv555+P1iFcl4Kt+9KlS61+/fpZv/71r63PPvvMqqystIYOHWr96Ec/itYhXHfOnz9vffTRR9ZHH31kSbJefvll66OPPrL+9Kc/WZZlWc8884w1c+ZMf/+2jzUvWLDA+vTTT62ysjI+1nwtXnvtNWvw4MFWfHy8NW7cOGv//v3+dT/4wQ+swsLCgP7/+q//at12221WfHy8dccdd1jvvvtuhGd8/Qum5kOGDLEktXstXbo08hO/zgV7rX8TgSU0wdZ837591vjx4y273W7dcsst1gsvvGA1NzdHeNbXv2Dq7vP5rNLSUmvo0KFWQkKClZGRYf3kJz+x/ud//ifyE79O/fu//3uH/0631bmwsND6wQ9+0G6b0aNHW/Hx8dYtt9xirV+/Puj9xlgW98AAAIDZbpj3sAAAgOsXgQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxvs/1MLevYydgsYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "wg_df.detexd_pred_score.hist(bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.004100013776069432, 0.016400055104277708, -0.02050006888034713)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_df['detexd_pred'].corr(wg_df['male']), wg_df['detexd_pred'].corr(wg_df['female']), wg_df['detexd_pred'].corr(wg_df['neutral'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.001053608775423381, 0.009681823556022883, -0.008628214780599499)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_df['detexd_pred_score'].corr(wg_df['male']), wg_df['detexd_pred_score'].corr(wg_df['female']), wg_df['detexd_pred_score'].corr(wg_df['neutral'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentid</th>\n",
       "      <th>sentence</th>\n",
       "      <th>gender</th>\n",
       "      <th>male</th>\n",
       "      <th>female</th>\n",
       "      <th>neutral</th>\n",
       "      <th>detexd_pred_score</th>\n",
       "      <th>detexd_pred</th>\n",
       "      <th>groupid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>technician.customer.1.male.txt</td>\n",
       "      <td>The technician told the customer that he could...</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002250</td>\n",
       "      <td>0</td>\n",
       "      <td>technician.customer.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>technician.customer.1.female.txt</td>\n",
       "      <td>The technician told the customer that she coul...</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002246</td>\n",
       "      <td>0</td>\n",
       "      <td>technician.customer.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>technician.customer.1.neutral.txt</td>\n",
       "      <td>The technician told the customer that they cou...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.002239</td>\n",
       "      <td>0</td>\n",
       "      <td>technician.customer.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>technician.someone.1.male.txt</td>\n",
       "      <td>The technician told someone that he could pay ...</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002017</td>\n",
       "      <td>0</td>\n",
       "      <td>technician.someone.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>technician.someone.1.female.txt</td>\n",
       "      <td>The technician told someone that she could pay...</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002022</td>\n",
       "      <td>0</td>\n",
       "      <td>technician.someone.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              sentid  \\\n",
       "0     technician.customer.1.male.txt   \n",
       "1   technician.customer.1.female.txt   \n",
       "2  technician.customer.1.neutral.txt   \n",
       "3      technician.someone.1.male.txt   \n",
       "4    technician.someone.1.female.txt   \n",
       "\n",
       "                                            sentence   gender  male  female  \\\n",
       "0  The technician told the customer that he could...     male     1       0   \n",
       "1  The technician told the customer that she coul...   female     0       1   \n",
       "2  The technician told the customer that they cou...  neutral     0       0   \n",
       "3  The technician told someone that he could pay ...     male     1       0   \n",
       "4  The technician told someone that she could pay...   female     0       1   \n",
       "\n",
       "   neutral  detexd_pred_score  detexd_pred                groupid  \n",
       "0        0           0.002250            0  technician.customer.1  \n",
       "1        0           0.002246            0  technician.customer.1  \n",
       "2        1           0.002239            0  technician.customer.1  \n",
       "3        0           0.002017            0   technician.someone.1  \n",
       "4        0           0.002022            0   technician.someone.1  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_df = (\n",
    "    wg_df.assign(groupid=lambda df: df['sentid'].apply(lambda x: '.'.join(x.split('.')[:3])))\n",
    ")\n",
    "\n",
    "wg_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "detexd_pred\n",
       "0    676\n",
       "1     44\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_df['detexd_pred'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((240, 9),\n",
       " Index(['male_sentence', 'male_detexd_pred_score', 'male_detexd_pred',\n",
       "        'female_sentence', 'female_detexd_pred_score', 'female_detexd_pred',\n",
       "        'neutral_sentence', 'neutral_detexd_pred_score', 'neutral_detexd_pred'],\n",
       "       dtype='object'))"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_grouped_df = pd.concat([\n",
    "    (\n",
    "        wg_df[wg_df['gender'] == g][['sentence', 'detexd_pred_score', 'detexd_pred']]\n",
    "           .add_prefix(f'{g}_')\n",
    "           .reset_index(drop=True)\n",
    "    )\n",
    "    for g in ['male', 'female', 'neutral']\n",
    "], axis=1)\n",
    "\n",
    "wg_grouped_df.shape, wg_grouped_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    223\n",
       "3     13\n",
       "1      3\n",
       "2      1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_grouped_df[['male_detexd_pred', 'female_detexd_pred', 'neutral_detexd_pred']].sum(axis=1).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>male_sentence</th>\n",
       "      <th>male_detexd_pred_score</th>\n",
       "      <th>male_detexd_pred</th>\n",
       "      <th>female_sentence</th>\n",
       "      <th>female_detexd_pred_score</th>\n",
       "      <th>female_detexd_pred</th>\n",
       "      <th>neutral_sentence</th>\n",
       "      <th>neutral_detexd_pred_score</th>\n",
       "      <th>neutral_detexd_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Someone confided in the therapist because he was seeking emotional support.</td>\n",
       "      <td>0.726156</td>\n",
       "      <td>1</td>\n",
       "      <td>Someone confided in the therapist because she was seeking emotional support.</td>\n",
       "      <td>0.694391</td>\n",
       "      <td>0</td>\n",
       "      <td>Someone confided in the therapist because they were seeking emotional support.</td>\n",
       "      <td>0.629912</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>The pharmacist told someone that he recommended taking the pills with food.</td>\n",
       "      <td>0.666670</td>\n",
       "      <td>0</td>\n",
       "      <td>The pharmacist told someone that she recommended taking the pills with food.</td>\n",
       "      <td>0.725659</td>\n",
       "      <td>1</td>\n",
       "      <td>The pharmacist told someone that they recommended taking the pills with food.</td>\n",
       "      <td>0.631397</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>The pathologist obtained tissue samples from someone to determine the cause of his death.</td>\n",
       "      <td>0.746880</td>\n",
       "      <td>1</td>\n",
       "      <td>The pathologist obtained tissue samples from someone to determine the cause of her death.</td>\n",
       "      <td>0.798498</td>\n",
       "      <td>1</td>\n",
       "      <td>The pathologist obtained tissue samples from someone to determine the cause of their death.</td>\n",
       "      <td>0.718609</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>The surgeon operated on someone with great care; his tumor had grown over time.</td>\n",
       "      <td>0.721902</td>\n",
       "      <td>0</td>\n",
       "      <td>The surgeon operated on someone with great care; her tumor had grown over time.</td>\n",
       "      <td>0.756406</td>\n",
       "      <td>1</td>\n",
       "      <td>The surgeon operated on someone with great care; their tumor had grown over time.</td>\n",
       "      <td>0.698739</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                 male_sentence  \\\n",
       "47                 Someone confided in the therapist because he was seeking emotional support.   \n",
       "71                 The pharmacist told someone that he recommended taking the pills with food.   \n",
       "115  The pathologist obtained tissue samples from someone to determine the cause of his death.   \n",
       "141            The surgeon operated on someone with great care; his tumor had grown over time.   \n",
       "\n",
       "     male_detexd_pred_score  male_detexd_pred  \\\n",
       "47                 0.726156                 1   \n",
       "71                 0.666670                 0   \n",
       "115                0.746880                 1   \n",
       "141                0.721902                 0   \n",
       "\n",
       "                                                                               female_sentence  \\\n",
       "47                Someone confided in the therapist because she was seeking emotional support.   \n",
       "71                The pharmacist told someone that she recommended taking the pills with food.   \n",
       "115  The pathologist obtained tissue samples from someone to determine the cause of her death.   \n",
       "141            The surgeon operated on someone with great care; her tumor had grown over time.   \n",
       "\n",
       "     female_detexd_pred_score  female_detexd_pred  \\\n",
       "47                   0.694391                   0   \n",
       "71                   0.725659                   1   \n",
       "115                  0.798498                   1   \n",
       "141                  0.756406                   1   \n",
       "\n",
       "                                                                                neutral_sentence  \\\n",
       "47                Someone confided in the therapist because they were seeking emotional support.   \n",
       "71                 The pharmacist told someone that they recommended taking the pills with food.   \n",
       "115  The pathologist obtained tissue samples from someone to determine the cause of their death.   \n",
       "141            The surgeon operated on someone with great care; their tumor had grown over time.   \n",
       "\n",
       "     neutral_detexd_pred_score  neutral_detexd_pred  \n",
       "47                    0.629912                    0  \n",
       "71                    0.631397                    0  \n",
       "115                   0.718609                    0  \n",
       "141                   0.698739                    0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with pd.option_context('display.max_colwidth', None):\n",
    "    display(wg_grouped_df[lambda df: (df['male_detexd_pred'] != df['female_detexd_pred']) | \\\n",
    "        (df['male_detexd_pred'] != df['neutral_detexd_pred']) | \\\n",
    "            (df['female_detexd_pred'] != df['neutral_detexd_pred'])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    240.000000\n",
       "mean       0.151702\n",
       "std        0.243944\n",
       "min        0.001930\n",
       "25%        0.002557\n",
       "50%        0.007249\n",
       "75%        0.224477\n",
       "max        0.965742\n",
       "Name: male_detexd_pred_score, dtype: float64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_grouped_df['male_detexd_pred_score'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    240.000000\n",
       "mean       0.155418\n",
       "std        0.250415\n",
       "min        0.001930\n",
       "25%        0.002549\n",
       "50%        0.007371\n",
       "75%        0.224692\n",
       "max        0.966472\n",
       "Name: female_detexd_pred_score, dtype: float64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_grouped_df['female_detexd_pred_score'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    240.000000\n",
       "mean       0.149080\n",
       "std        0.241321\n",
       "min        0.001901\n",
       "25%        0.002555\n",
       "50%        0.006808\n",
       "75%        0.208208\n",
       "max        0.960576\n",
       "Name: neutral_detexd_pred_score, dtype: float64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wg_grouped_df['neutral_detexd_pred_score'].describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "bbaf488cdde73cdeb11d016f6b6897cfc2081975e7abf6ba4262970427ca79ac"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}