examples/DKT/prepare_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "downloader, INFO ../../data/2009_skill_builder_data_corrected.zip already exists. Send resume request after 9084422 bytes\n",
      "downloader, INFO http://base.ustc.edu.cn/data/ASSISTment/2009_skill_builder_data_corrected.zip is saved as ../../data/2009_skill_builder_data_corrected.zip\n",
      "downloader, WARNING Range not support. Redownloading...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading 100.00% : 9084928 | 9084422"
     ]
    },
    {
     "data": {
      "text/plain": "'../../data'"
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from EduData import get_data\n",
    "\n",
    "get_data(\"assistment-2009-2010-skill\", \"../../data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pandas as pd\n",
    "import tqdm\n",
    "import numpy as np\n",
    "\n",
    "data = pd.read_csv(\n",
    "    '../../data/2009_skill_builder_data_corrected/skill_builder_data_corrected.csv',\n",
    "    usecols=['order_id', 'user_id', 'sequence_id', 'skill_id', 'correct']\n",
    ").dropna(subset=['skill_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of skills: 123\n"
     ]
    }
   ],
   "source": [
    "raw_question = data.skill_id.unique().tolist()\n",
    "num_skill = len(raw_question)\n",
    "\n",
    "# question id from 0 to (num_skill - 1)\n",
    "questions = { p: i for i, p in enumerate(raw_question) }\n",
    "\n",
    "print(\"number of skills: %d\" % num_skill)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "parse student sequence:\t: 100%|██████████| 4163/4163 [00:04<00:00, 951.53it/s] \n"
     ]
    }
   ],
   "source": [
    "def parse_all_seq(students):\n",
    "    all_sequences = []\n",
    "    for student_id in tqdm.tqdm(students, 'parse student sequence:\\t'):\n",
    "        student_sequence = parse_student_seq(data[data.user_id == student_id])\n",
    "        all_sequences.extend([student_sequence])\n",
    "    return all_sequences\n",
    "\n",
    "\n",
    "def parse_student_seq(student):\n",
    "    seq = student.sort_values('order_id')\n",
    "    q = [questions[q] for q in seq.skill_id.tolist()]\n",
    "    a = seq.correct.tolist()\n",
    "    return q, a\n",
    "\n",
    "\n",
    "# [(question_sequence_0, answer_sequence_0), ..., (question_sequence_n, answer_sequence_n)]\n",
    "sequences = parse_all_seq(data.user_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_test_split(data, train_size=.7, shuffle=True):\n",
    "    if shuffle:\n",
    "        random.shuffle(data)\n",
    "    boundary = round(len(data) * train_size)\n",
    "    return data[: boundary], data[boundary:]\n",
    "\n",
    "\n",
    "train_sequences, test_sequences = train_test_split(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "write into file: 100%|██████████| 2914/2914 [00:00<00:00, 11200.62it/s]\n",
      "write into file: 100%|██████████| 1249/1249 [00:00<00:00, 27617.82it/s]\n"
     ]
    }
   ],
   "source": [
    "def sequences2tl(sequences, trgpath):\n",
    "    with open(trgpath, 'a', encoding='utf8') as f:\n",
    "        for seq in tqdm.tqdm(sequences, 'write into file: '):\n",
    "            questions, answers = seq\n",
    "            seq_len = len(questions)\n",
    "            f.write(str(seq_len) + '\\n')\n",
    "            f.write(','.join([str(q) for q in questions]) + '\\n')\n",
    "            f.write(','.join([str(a) for a in answers]) + '\\n')\n",
    "\n",
    "\n",
    "# save triple line format for other tasks\n",
    "sequences2tl(train_sequences, '../../data/2009_skill_builder_data_corrected/train.txt')\n",
    "sequences2tl(test_sequences, '../../data/2009_skill_builder_data_corrected/test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "convert to one-hot format: 100%|██████████| 145/145 [00:02<00:00, 67.81it/s]\n",
      "convert to one-hot format: 100%|██████████| 62/62 [00:00<00:00, 173.26it/s]\n"
     ]
    }
   ],
   "source": [
    "MAX_STEP = 50\n",
    "NUM_QUESTIONS = num_skill\n",
    "\n",
    "\n",
    "def encode_onehot(sequences, max_step, num_questions):\n",
    "    result = []\n",
    "\n",
    "    for q, a in tqdm.tqdm(sequences, 'convert to one-hot format: '):\n",
    "        length = len(q)\n",
    "        # append questions' and answers' length to an integer multiple of max_step\n",
    "        mod = 0 if length % max_step == 0 else (max_step - length % max_step)\n",
    "        onehot = np.zeros(shape=[length + mod, 2 * num_questions])\n",
    "        for i, q_id in enumerate(q):\n",
    "            index = int(q_id if a[i] > 0 else q_id + num_questions)\n",
    "            onehot[i][index] = 1\n",
    "        result = np.append(result, onehot)\n",
    "    \n",
    "    return result.reshape(-1, max_step, 2 * num_questions)\n",
    "\n",
    "\n",
    "# reduce the amount of data for example running faster\n",
    "percentage = 0.05\n",
    "train_data = encode_onehot(train_sequences[: int(len(train_sequences) * percentage)], MAX_STEP, NUM_QUESTIONS)\n",
    "test_data = encode_onehot(test_sequences[: int(len(test_sequences) * percentage)], MAX_STEP, NUM_QUESTIONS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save onehot data\n",
    "np.save('../../data/2009_skill_builder_data_corrected/train_data.npy', train_data)\n",
    "np.save('../../data/2009_skill_builder_data_corrected/test_data.npy', test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}