Examples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import necessary modules \n",
    "# please run this everytime you open it\n",
    "import pandas as pd \n",
    "import numpy as np \n",
    "import matplotlib as plt \n",
    "import seaborn as sns\n",
    "import re\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run Validation notebook\n",
    "%run Validation.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Specialists"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example of get_topic()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Quote Source Relevance'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.read_csv(\"Specialists data/Covid_SourceRelevancev1-2020-10-04T1838-DataHunt.csv\")\n",
    "get_topic(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example to_reliability()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 0 rows where all values are missing\n",
      "There are 0 cols where all values are missing\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([nan, nan, nan, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 2, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, 2, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 5, nan, nan,\n",
       "       1, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, 3, nan, 3, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 1, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan], dtype=object)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array = to_reliability(\"Specialists data\", \"Covid_SourceRelevancev1-2020-10-04T1838-DataHunt.csv\", \"T1.Q2\")\n",
    "array[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quiz_task_uuid</th>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th>question_label</th>\n",
       "      <th>answer_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9d4ac736-52b7-40df-b2a9-6164fa081960</td>\n",
       "      <td>00f548b7-6b63-4b47-828e-8e416b6ca0e2</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>9d4ac736-52b7-40df-b2a9-6164fa081960</td>\n",
       "      <td>e44f5799-6915-45fd-874e-060f6afedcb9</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>9d4ac736-52b7-40df-b2a9-6164fa081960</td>\n",
       "      <td>fed45769-aa61-4b8b-847e-a9c799acce15</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>9d4ac736-52b7-40df-b2a9-6164fa081960</td>\n",
       "      <td>fb4ae4cc-234b-484a-b541-a8e3a9254a24</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>9d4ac736-52b7-40df-b2a9-6164fa081960</td>\n",
       "      <td>3bb03f58-5f09-4414-b54e-43a814b9b09a</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          quiz_task_uuid  \\\n",
       "1   9d4ac736-52b7-40df-b2a9-6164fa081960   \n",
       "18  9d4ac736-52b7-40df-b2a9-6164fa081960   \n",
       "40  9d4ac736-52b7-40df-b2a9-6164fa081960   \n",
       "53  9d4ac736-52b7-40df-b2a9-6164fa081960   \n",
       "73  9d4ac736-52b7-40df-b2a9-6164fa081960   \n",
       "\n",
       "                        contributor_uuid question_label answer_label  \n",
       "1   00f548b7-6b63-4b47-828e-8e416b6ca0e2          T1.Q2     T1.Q2.A1  \n",
       "18  e44f5799-6915-45fd-874e-060f6afedcb9          T1.Q2     T1.Q2.A2  \n",
       "40  fed45769-aa61-4b8b-847e-a9c799acce15          T1.Q2     T1.Q2.A2  \n",
       "53  fb4ae4cc-234b-484a-b541-a8e3a9254a24          T1.Q2     T1.Q2.A1  \n",
       "73  3bb03f58-5f09-4414-b54e-43a814b9b09a          T1.Q2     T1.Q2.A1  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load the example data \n",
    "relevance = pd.read_csv(\"Specialists data/Covid_SourceRelevancev1-2020-10-04T1838-DataHunt.csv\")\n",
    "# select certain columns \n",
    "cols = [\"quiz_task_uuid\", \"contributor_uuid\", \"question_label\", \"answer_label\"]\n",
    "relevance_q2 = relevance[relevance[\"question_label\"] == \"T1.Q2\"]\n",
    "relevance_q2 = relevance_q2.reindex(columns=cols)\n",
    "relevance_q2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Construct the reliability matrix, the left column is the contributor_uuid, each unit -> quiz_taskrun_uuid**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>quiz_task_uuid</th>\n",
       "      <th>00ef0777-da36-47b0-94d4-135eb520bcfb</th>\n",
       "      <th>0214dcdb-838e-4947-8a8e-7cc8f8f2fce6</th>\n",
       "      <th>08e6aae7-56c5-4c0f-8517-d2a7f2bc5895</th>\n",
       "      <th>1b4d23d1-fa87-44fe-b5b2-1a5909246b66</th>\n",
       "      <th>23699e71-779c-4a97-81aa-e0bdc8c69432</th>\n",
       "      <th>3032089b-c0e4-4f39-ac05-98e135113625</th>\n",
       "      <th>4479a237-11ec-4a69-bab1-090b44fb62a9</th>\n",
       "      <th>65fc3c92-5aef-4fef-a3c4-56145a077a33</th>\n",
       "      <th>72f6d8cc-9f38-4040-b08c-c9623db19f41</th>\n",
       "      <th>7995a0fe-1c4a-4d51-9064-1897e7fdf297</th>\n",
       "      <th>...</th>\n",
       "      <th>a84b3b17-d59f-4350-815b-e7793ff1f76f</th>\n",
       "      <th>b27a22c1-bea6-4620-abfa-428d27316d2a</th>\n",
       "      <th>b2846191-15d5-4129-983d-ee94ad66e9c9</th>\n",
       "      <th>b4ae9d51-4923-4765-a6b2-049afe8ec571</th>\n",
       "      <th>b4d3861f-5db7-4def-b3d8-14ab384985b5</th>\n",
       "      <th>bd930f84-9d64-49b5-a922-eb99ae49239f</th>\n",
       "      <th>c00936dc-7aae-4082-9e2e-3c86c7e17956</th>\n",
       "      <th>d97d6501-b50f-4f8c-a3c5-59cd05a9b752</th>\n",
       "      <th>e399f1ca-2f2c-4803-ad87-6f5fe090ee33</th>\n",
       "      <th>e9de370c-d418-4200-b151-77404581a518</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00f548b7-6b63-4b47-828e-8e416b6ca0e2</th>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>T1.Q2.A1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>06e1eba7-c695-4495-85a7-2fb33ef9412c</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0b2ee377-65f2-4ac1-98da-5bc093fee7aa</th>\n",
       "      <td>T1.Q2.A6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A6</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0cdf31d0-1966-4f69-a0c8-c483788b5256</th>\n",
       "      <td>T1.Q2.A2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "quiz_task_uuid                       00ef0777-da36-47b0-94d4-135eb520bcfb  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                             T1.Q2.A5   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                             T1.Q2.A6   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                             T1.Q2.A2   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       0214dcdb-838e-4947-8a8e-7cc8f8f2fce6  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       08e6aae7-56c5-4c0f-8517-d2a7f2bc5895  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       1b4d23d1-fa87-44fe-b5b2-1a5909246b66  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       23699e71-779c-4a97-81aa-e0bdc8c69432  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       3032089b-c0e4-4f39-ac05-98e135113625  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       4479a237-11ec-4a69-bab1-090b44fb62a9  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                             T1.Q2.A5   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                             T1.Q2.A6   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                             T1.Q2.A5   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       65fc3c92-5aef-4fef-a3c4-56145a077a33  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                             T1.Q2.A1   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                             T1.Q2.A5   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       72f6d8cc-9f38-4040-b08c-c9623db19f41  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       7995a0fe-1c4a-4d51-9064-1897e7fdf297  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                             T1.Q2.A5   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                        ...  \\\n",
       "contributor_uuid                      ...   \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2  ...   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c  ...   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa  ...   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256  ...   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6  ...   \n",
       "\n",
       "quiz_task_uuid                       a84b3b17-d59f-4350-815b-e7793ff1f76f  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       b27a22c1-bea6-4620-abfa-428d27316d2a  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                             T1.Q2.A5   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                             T1.Q2.A5   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       b2846191-15d5-4129-983d-ee94ad66e9c9  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                             T1.Q2.A5   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                             T1.Q2.A7   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       b4ae9d51-4923-4765-a6b2-049afe8ec571  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       b4d3861f-5db7-4def-b3d8-14ab384985b5  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                             T1.Q2.A3   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       bd930f84-9d64-49b5-a922-eb99ae49239f  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       c00936dc-7aae-4082-9e2e-3c86c7e17956  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       d97d6501-b50f-4f8c-a3c5-59cd05a9b752  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                             T1.Q2.A3   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                             T1.Q2.A5   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       e399f1ca-2f2c-4803-ad87-6f5fe090ee33  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "\n",
       "quiz_task_uuid                       e9de370c-d418-4200-b151-77404581a518  \n",
       "contributor_uuid                                                           \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN  \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN  \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN  \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN  \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.pivot_table(relevance_q2, values='answer_label', index='contributor_uuid', columns='quiz_task_uuid', aggfunc=select)\n",
    "# drop nan rows \n",
    "df = df.dropna(axis=1, how='all')\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['T1.Q2.A5', nan, nan, nan, nan, nan, 'T1.Q2.A5', 'T1.Q2.A1', nan,\n",
       "       nan, 'T1.Q2.A2', 'T1.Q2.A5', 'T1.Q2.A2', nan, nan, 'T1.Q2.A1', nan,\n",
       "       'T1.Q2.A5', 'T1.Q2.A5', nan, 'T1.Q2.A3', nan, nan, 'T1.Q2.A3', nan,\n",
       "       nan], dtype=object)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array = df.to_numpy()\n",
    "array[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate through all values in array and convert string to int \n",
    "for i in range(array.shape[0]):\n",
    "    for j in range(array.shape[1]):\n",
    "        cur = array[i][j]\n",
    "        # if the current value is a string\n",
    "        if not pd.isnull(cur):\n",
    "            array[i][j] = int(array[i][j][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0    1    2    3    4    5    6    7    8    9   ...   16   17   18   19  \\\n",
       "0    5  NaN  NaN  NaN  NaN  NaN    5    1  NaN  NaN  ...  NaN    5    5  NaN   \n",
       "1  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  ...  NaN  NaN  NaN  NaN   \n",
       "2    6  NaN  NaN  NaN  NaN  NaN    6    5  NaN    5  ...  NaN  NaN    7  NaN   \n",
       "3    2  NaN  NaN  NaN  NaN  NaN    5  NaN  NaN  NaN  ...  NaN    5  NaN  NaN   \n",
       "4  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  ...  NaN  NaN  NaN  NaN   \n",
       "\n",
       "    20   21   22   23   24   25  \n",
       "0    3  NaN  NaN    3  NaN  NaN  \n",
       "1  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "2  NaN  NaN  NaN    5  NaN  NaN  \n",
       "3  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "4  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(array)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.191812762\n"
     ]
    }
   ],
   "source": [
    "print(round(alpha(reliability_data=df, level_of_measurement='nominal'), 10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example of ordinal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example file: Evidence <br>\n",
    "Question: 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "evidence = pd.read_csv(\"Specialists data/Covid_Evidencev1-2020-10-04T1836-DataHunt.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01752"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_alpha(\"Specialists data\", \"Covid_Evidencev1-2020-10-04T1836-DataHunt.csv\", \"T1.Q11\", \"ordinal\", 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example of checklist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example file: Covid_Reasoningv1-2020-10-04T1803-DataHunt.csv <br>\n",
    "Question: 1 (cl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "reasoning = pd.read_csv(\"Specialists data/Covid_Reasoningv1-2020-10-04T1803-DataHunt.csv\")\n",
    "reasoning = reasoning[reasoning[\"question_label\"] == \"T1.Q1\"]\n",
    "reasoning = reasoning.reindex(columns=cols)\n",
    "# reasoning.to_excel(\"example1.xlsx\")\n",
    "example = pd.get_dummies(reasoning, columns = [\"answer_label\"])\n",
    "# example.to_excel(\"example2.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_three = example.groupby([\"quiz_task_uuid\", \"contributor_uuid\"]).agg(sum)\n",
    "example_three = example_three.reset_index()\n",
    "# example_three.to_excel(\"example_three.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can only concatenate list (not \"str\") to list",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-93-c877c27ec677>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m example_four = pd.melt(example_three, id_vars=[\"quiz_task_uuid\", \"contributor_uuid\"], \n\u001b[1;32m----> 2\u001b[1;33m                                   \u001b[0mvar_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m\"quiz_task_uuid\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"choice\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m                                   value_name = \"value\")\n\u001b[0;32m      4\u001b[0m \u001b[0mexample_four\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexample_four\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"quiz_task_uuid\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"contributor_uuid\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"choice\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mexample_four\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_excel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"example_four.xlsx\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: can only concatenate list (not \"str\") to list"
     ]
    }
   ],
   "source": [
    "example_four = pd.melt(example_three, id_vars=[\"quiz_task_uuid\", \"contributor_uuid\"], \n",
    "                                  var_name = \"choice\", \n",
    "                                  value_name = \"value\")\n",
    "example_four = example_four.sort_values(by=[\"quiz_task_uuid\", \"contributor_uuid\", \"choice\"])\n",
    "# example_four.to_excel(\"example_four.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quiz_task_uuid</th>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th>choice</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>085cde3c-97c3-44a4-a773-37f7d5704107</td>\n",
       "      <td>00f548b7-6b63-4b47-828e-8e416b6ca0e2</td>\n",
       "      <td>085cde answer_label_T1.Q1.A1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>085cde3c-97c3-44a4-a773-37f7d5704107</td>\n",
       "      <td>00f548b7-6b63-4b47-828e-8e416b6ca0e2</td>\n",
       "      <td>085cde answer_label_T1.Q1.A2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272</th>\n",
       "      <td>085cde3c-97c3-44a4-a773-37f7d5704107</td>\n",
       "      <td>00f548b7-6b63-4b47-828e-8e416b6ca0e2</td>\n",
       "      <td>085cde answer_label_T1.Q1.A3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>408</th>\n",
       "      <td>085cde3c-97c3-44a4-a773-37f7d5704107</td>\n",
       "      <td>00f548b7-6b63-4b47-828e-8e416b6ca0e2</td>\n",
       "      <td>085cde answer_label_T1.Q1.A4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544</th>\n",
       "      <td>085cde3c-97c3-44a4-a773-37f7d5704107</td>\n",
       "      <td>00f548b7-6b63-4b47-828e-8e416b6ca0e2</td>\n",
       "      <td>085cde answer_label_T1.Q1.A5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           quiz_task_uuid  \\\n",
       "0    085cde3c-97c3-44a4-a773-37f7d5704107   \n",
       "136  085cde3c-97c3-44a4-a773-37f7d5704107   \n",
       "272  085cde3c-97c3-44a4-a773-37f7d5704107   \n",
       "408  085cde3c-97c3-44a4-a773-37f7d5704107   \n",
       "544  085cde3c-97c3-44a4-a773-37f7d5704107   \n",
       "\n",
       "                         contributor_uuid                        choice  value  \n",
       "0    00f548b7-6b63-4b47-828e-8e416b6ca0e2  085cde answer_label_T1.Q1.A1      0  \n",
       "136  00f548b7-6b63-4b47-828e-8e416b6ca0e2  085cde answer_label_T1.Q1.A2      0  \n",
       "272  00f548b7-6b63-4b47-828e-8e416b6ca0e2  085cde answer_label_T1.Q1.A3      1  \n",
       "408  00f548b7-6b63-4b47-828e-8e416b6ca0e2  085cde answer_label_T1.Q1.A4      0  \n",
       "544  00f548b7-6b63-4b47-828e-8e416b6ca0e2  085cde answer_label_T1.Q1.A5      0  "
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_five = example_four.copy()\n",
    "example_five[\"choice\"] = example_five[\"quiz_task_uuid\"].str.slice(start=0, stop=6) + \" \" + example_five[\"choice\"]\n",
    "example_five.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_six = pd.pivot_table(example_five, values='value', index='contributor_uuid', \n",
    "                          columns='choice', \n",
    "                          aggfunc=select)\n",
    "example_six.to_excel(\"example_6.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>choice</th>\n",
       "      <th>085cde answer_label_T1.Q1.A1</th>\n",
       "      <th>085cde answer_label_T1.Q1.A2</th>\n",
       "      <th>085cde answer_label_T1.Q1.A3</th>\n",
       "      <th>085cde answer_label_T1.Q1.A4</th>\n",
       "      <th>085cde answer_label_T1.Q1.A5</th>\n",
       "      <th>085cde answer_label_T1.Q1.A6</th>\n",
       "      <th>15ef35 answer_label_T1.Q1.A1</th>\n",
       "      <th>15ef35 answer_label_T1.Q1.A2</th>\n",
       "      <th>15ef35 answer_label_T1.Q1.A3</th>\n",
       "      <th>15ef35 answer_label_T1.Q1.A4</th>\n",
       "      <th>...</th>\n",
       "      <th>fce4b7 answer_label_T1.Q1.A3</th>\n",
       "      <th>fce4b7 answer_label_T1.Q1.A4</th>\n",
       "      <th>fce4b7 answer_label_T1.Q1.A5</th>\n",
       "      <th>fce4b7 answer_label_T1.Q1.A6</th>\n",
       "      <th>feaaf7 answer_label_T1.Q1.A1</th>\n",
       "      <th>feaaf7 answer_label_T1.Q1.A2</th>\n",
       "      <th>feaaf7 answer_label_T1.Q1.A3</th>\n",
       "      <th>feaaf7 answer_label_T1.Q1.A4</th>\n",
       "      <th>feaaf7 answer_label_T1.Q1.A5</th>\n",
       "      <th>feaaf7 answer_label_T1.Q1.A6</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00f548b7-6b63-4b47-828e-8e416b6ca0e2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1119f9ca-0dce-4a9e-b71e-5983833e8bbe</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172e9a38-d3e4-453e-8556-18bbd0fcb07e</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214e3374-4279-4748-80c8-ea8a76ad7cbc</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343ecb7a-aeb5-4b21-9e9c-df545dafba99</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 174 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "choice                                085cde answer_label_T1.Q1.A1  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                085cde answer_label_T1.Q1.A2  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           1.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                085cde answer_label_T1.Q1.A3  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           1.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                085cde answer_label_T1.Q1.A4  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                085cde answer_label_T1.Q1.A5  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                085cde answer_label_T1.Q1.A6  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                15ef35 answer_label_T1.Q1.A1  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           0.0   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                15ef35 answer_label_T1.Q1.A2  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           0.0   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                15ef35 answer_label_T1.Q1.A3  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           0.0   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                15ef35 answer_label_T1.Q1.A4  ...  \\\n",
       "contributor_uuid                                                    ...   \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0  ...   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN  ...   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           0.0  ...   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0  ...   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN  ...   \n",
       "\n",
       "choice                                fce4b7 answer_label_T1.Q1.A3  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                fce4b7 answer_label_T1.Q1.A4  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           1.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           1.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                fce4b7 answer_label_T1.Q1.A5  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                fce4b7 answer_label_T1.Q1.A6  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                feaaf7 answer_label_T1.Q1.A1  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                feaaf7 answer_label_T1.Q1.A2  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                feaaf7 answer_label_T1.Q1.A3  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           1.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           1.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                feaaf7 answer_label_T1.Q1.A4  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                feaaf7 answer_label_T1.Q1.A5  \\\n",
       "contributor_uuid                                                     \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0   \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN   \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN   \n",
       "\n",
       "choice                                feaaf7 answer_label_T1.Q1.A6  \n",
       "contributor_uuid                                                    \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                           0.0  \n",
       "1119f9ca-0dce-4a9e-b71e-5983833e8bbe                           NaN  \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                           NaN  \n",
       "214e3374-4279-4748-80c8-ea8a76ad7cbc                           0.0  \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                           NaN  \n",
       "\n",
       "[5 rows x 174 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_six.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = example_six.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  1., ...,  0.,  0.,  0.],\n",
       "       [nan, nan, nan, ..., nan, nan, nan],\n",
       "       [nan, nan, nan, ..., nan, nan, nan],\n",
       "       ...,\n",
       "       [nan, nan, nan, ..., nan, nan, nan],\n",
       "       [nan, nan, nan, ..., nan, nan, nan],\n",
       "       [nan, nan, nan, ..., nan, nan, nan]])"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.265466\n"
     ]
    }
   ],
   "source": [
    "print(round(alpha(reliability_data=array, level_of_measurement='nominal'), 6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## compute_alpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 0 rows where all values are missing\n",
      "There are 0 cols where all values are missing\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>248</th>\n",
       "      <th>249</th>\n",
       "      <th>250</th>\n",
       "      <th>251</th>\n",
       "      <th>252</th>\n",
       "      <th>253</th>\n",
       "      <th>254</th>\n",
       "      <th>255</th>\n",
       "      <th>256</th>\n",
       "      <th>257</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 258 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0    1    2    3    4    5    6    7    8    9    ...  248  249  250  251  \\\n",
       "0  NaN  NaN  NaN    5  NaN  NaN  NaN  NaN  NaN  NaN  ...  NaN  NaN  NaN  NaN   \n",
       "1  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  ...  NaN  NaN  NaN  NaN   \n",
       "2  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  ...  NaN  NaN  NaN  NaN   \n",
       "3  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  ...  NaN  NaN  NaN  NaN   \n",
       "4  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  ...  NaN  NaN  NaN  NaN   \n",
       "\n",
       "   252  253  254  255  256  257  \n",
       "0  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "1  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "2  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "3  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "4  NaN  NaN  NaN  NaN  NaN  NaN  \n",
       "\n",
       "[5 rows x 258 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0.]]\n",
      "[0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[[-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]]\n",
      "[[0. 1. 1. 1. 1. 1. 1. 1.]\n",
      " [1. 0. 1. 1. 1. 1. 1. 1.]\n",
      " [1. 1. 0. 1. 1. 1. 1. 1.]\n",
      " [1. 1. 1. 0. 1. 1. 1. 1.]\n",
      " [1. 1. 1. 1. 0. 1. 1. 1.]\n",
      " [1. 1. 1. 1. 1. 0. 1. 1.]\n",
      " [1. 1. 1. 1. 1. 1. 0. 1.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0.]]\n",
      "[[-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]\n",
      " [-0. -0. -0. -0. -0. -0. -0. -0.]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thefi\\Desktop\\PE-Validation-Fall-20\\krippendorff.py:283: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_alpha(\"Specialists data\", \"Covid_SourceRelevancev1-2020-10-04T1838-DataHunt.csv\", \"T1.Q2\", \"nominal\", 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 0 rows where all values are missing\n",
      "There are 0 cols where all values are missing\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0    1    2    3    4    5    6    7    8    9    10   11   12   13   14  \\\n",
       "0  NaN  NaN    3  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       "1  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN    3  NaN  NaN  NaN  NaN  NaN   \n",
       "2  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       "3  NaN  NaN  NaN  NaN  NaN    2  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       "4  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       "\n",
       "    15   16   17  \n",
       "0  NaN  NaN  NaN  \n",
       "1  NaN  NaN  NaN  \n",
       "2  NaN    2  NaN  \n",
       "3  NaN  NaN  NaN  \n",
       "4  NaN  NaN    3  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0.]\n",
      " [0. 0. 0.]\n",
      " [0. 0. 0.]]\n",
      "[0. 0. 0.]\n",
      "[[-0. -0. -0.]\n",
      " [-0. -0. -0.]\n",
      " [-0. -0. -0.]]\n",
      "[[0. 1. 1.]\n",
      " [1. 0. 1.]\n",
      " [1. 1. 0.]]\n",
      "[[-0. -0. -0.]\n",
      " [-0. -0. -0.]\n",
      " [-0. -0. -0.]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_alpha(\"Specialists data\", \"Covid_Evidencev1-2020-10-04T1836-DataHunt.csv\", \"T1.Q6\", \"nominal\", 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 0 rows where all values are missing\n",
      "There are 0 cols where all values are missing\n",
      "[array(['T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, array([], dtype=object), nan, nan, nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A3', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, array([], dtype=object), 'T1.Q6.A2', nan, nan, nan, nan,\n",
      "       nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, 'T1.Q6.A1',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, 'T1.Q6.A1', 'T1.Q6.A1', nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       nan, nan, nan, array([], dtype=object), nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       array([], dtype=object), nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A2',\n",
      "       nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, array([], dtype=object), nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A2', nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1',\n",
      "       nan, nan, nan, nan, 'T1.Q6.A2', 'T1.Q6.A1', nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, 'T1.Q6.A2', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan,\n",
      "       array([], dtype=object), nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, 'T1.Q6.A1', 'T1.Q6.A1', nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, array([], dtype=object), nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, 'T1.Q6.A3', nan, nan, nan, 'T1.Q6.A1', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, 'T1.Q6.A1', nan, nan, nan, array([], dtype=object), nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan,\n",
      "       nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, 'T1.Q6.A2', nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, 'T1.Q6.A2', nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, array([], dtype=object), nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A2',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A2', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, 'T1.Q6.A1', nan, 'T1.Q6.A1', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1',\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, 'T1.Q6.A1'], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A2', nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1',\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       array([], dtype=object), nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, 'T1.Q6.A1', nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan,\n",
      "       nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A2', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       array([], dtype=object), array([], dtype=object), nan, nan, nan,\n",
      "       nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A2', nan, nan, nan],\n",
      "      dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A2',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, array([], dtype=object),\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, array([], dtype=object), 'T1.Q6.A1', nan, nan, 'T1.Q6.A2',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],\n",
      "      dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, array([], dtype=object), nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],\n",
      "      dtype=object), array([nan, nan, nan, nan, nan, 'T1.Q6.A2', nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan], dtype=object), array([nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan,\n",
      "       nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, 'T1.Q6.A1',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, 'T1.Q6.A1',\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A2', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A2', nan], dtype=object), array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       array([], dtype=object), nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, 'T1.Q6.A1', nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
      "       nan, nan, nan, nan, nan], dtype=object)]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thefi\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_alpha(\"Specialists data\", \"Covid_Probabilityv1-2020-10-04T1837-DataHunt.csv\", \"T1.Q6\", \"nominal\", 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, 4, 1, 2, np.nan, np.nan, np.nan],\n",
    "     [1, 2, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, 3],\n",
    "     [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],\n",
    "     [3, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 1. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [1. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]]\n",
      "[1. 0. 1. 0.]\n",
      "[[0. 0. 1. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [1. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]]\n",
      "[[0. 1. 1. 1.]\n",
      " [1. 0. 1. 1.]\n",
      " [1. 1. 0. 1.]\n",
      " [1. 1. 1. 0.]]\n",
      "[[0. 0. 1. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [1. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]]\n",
      "0.0\n"
     ]
    }
   ],
   "source": [
    "print(round(alpha(reliability_data=a, level_of_measurement='nominal'), 10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Triager Tasks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example of Triager_split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Triager Tranformation Done! Ready to be imported to uAlpha!\n"
     ]
    }
   ],
   "source": [
    "triager_split(\"Triager data\", \"Triager output\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example of resolving overlap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th>topic_name</th>\n",
       "      <th>start_pos</th>\n",
       "      <th>blank</th>\n",
       "      <th>end_pos</th>\n",
       "      <th>article_text_length</th>\n",
       "      <th>article_number</th>\n",
       "      <th>created</th>\n",
       "      <th>cumulative</th>\n",
       "      <th>start_pos_adjusted</th>\n",
       "      <th>end_pos_adjusted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>136</td>\n",
       "      <td>NaN</td>\n",
       "      <td>287</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4021</td>\n",
       "      <td>4172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>563</td>\n",
       "      <td>NaN</td>\n",
       "      <td>776</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4448</td>\n",
       "      <td>4661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>651</td>\n",
       "      <td>NaN</td>\n",
       "      <td>667</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4536</td>\n",
       "      <td>4552</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  contributor_uuid  topic_name  start_pos  blank  end_pos  \\\n",
       "0           aac18e           2        136    NaN      287   \n",
       "1           aac18e           2        563    NaN      776   \n",
       "2           aac18e           2        651    NaN      667   \n",
       "\n",
       "   article_text_length  article_number                     created  \\\n",
       "0                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "1                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "2                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "\n",
       "   cumulative  start_pos_adjusted  end_pos_adjusted  \n",
       "0        5755                4021              4172  \n",
       "1        5755                4448              4661  \n",
       "2        5755                4536              4552  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.read_csv(\"example.csv\")\n",
    "a.drop(a.columns[0], axis=1, inplace=True)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th>topic_name</th>\n",
       "      <th>start_pos</th>\n",
       "      <th>blank</th>\n",
       "      <th>end_pos</th>\n",
       "      <th>article_text_length</th>\n",
       "      <th>article_number</th>\n",
       "      <th>created</th>\n",
       "      <th>cumulative</th>\n",
       "      <th>start_pos_adjusted</th>\n",
       "      <th>end_pos_adjusted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>136</td>\n",
       "      <td>NaN</td>\n",
       "      <td>287</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4021</td>\n",
       "      <td>4172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>563</td>\n",
       "      <td>NaN</td>\n",
       "      <td>776</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4448</td>\n",
       "      <td>4661</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  contributor_uuid  topic_name  start_pos  blank  end_pos  \\\n",
       "0           aac18e           2        136    NaN      287   \n",
       "1           aac18e           2        563    NaN      776   \n",
       "\n",
       "   article_text_length  article_number                     created  \\\n",
       "0                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "1                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "\n",
       "   cumulative  start_pos_adjusted  end_pos_adjusted  \n",
       "0        5755                4021              4172  \n",
       "1        5755                4448              4661  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = resolve_overlapping(a)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "detect_overlapping(\"Triager output\", \"Covid-Triager-Arguments.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example of removing a row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    first\n",
       "23      1\n",
       "24      2\n",
       "25      3\n",
       "26      4\n",
       "27      5"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.DataFrame(index=np.array([23, 24, 25, 26, 27]), data={\"first\": [1, 2, 3, 4, 5]})\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    first\n",
       "23      1\n",
       "24      2\n",
       "26      4\n",
       "27      5"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.drop(25, inplace=True)\n",
    "a"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}