A_ML Model 3 Inputs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a82dc099-0fb8-485f-9520-cd514ded8580",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "916ecf61-acb8-4647-9220-c71a4c91fc02",
   "metadata": {},
   "outputs": [],
   "source": [
    "#split df_full into df for training/testing and new_df for evaluation\n",
    "df_full = pd.read_csv('random subset of data files/NP_ENG_EOC_FULL.csv')\n",
    "df = df_full.iloc[0:8000]\n",
    "new_df = df_full.iloc[8001:9238]\n",
    "new_df = new_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "14baa414-665b-410c-b1d1-9a049ce6fcb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reorganize columns df\n",
    "columns = list(df.columns)\n",
    "columns.remove('EOC')\n",
    "columns.append('EOC')\n",
    "\n",
    "# Reassign the DataFrame to follow the new column order\n",
    "df = df[columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "999947dc-150e-4c58-8dbd-7f7aa9f2d538",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming 'df' is your pandas DataFrame with the data\n",
    "# Step 1: Preprocessing\n",
    "# Separate features and labels\n",
    "X = df[['num_passed', 'engaged', 'length']].values  # Features\n",
    "y = df['EOC'].values  # Labels (EOC accuracy)\n",
    "y_tensor = torch.tensor(y, dtype=torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "88382206-d28f-40cc-8758-2c81cf14dc96",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into training and testing set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Normalize features\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "229e06ad-0c26-4575-9cdd-71c60bd53133",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to PyTorch tensors\n",
    "X_train_tensor = torch.tensor(X_train, dtype=torch.float32)\n",
    "y_train_tensor = torch.tensor(y_train, dtype=torch.float32)\n",
    "X_test_tensor = torch.tensor(X_test, dtype=torch.float32)\n",
    "y_test_tensor = torch.tensor(y_test, dtype=torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1437dafe-1a8c-428d-b2a0-e26d98c9180f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 2: Model Definition\n",
    "class RegressionModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(RegressionModel, self).__init__()\n",
    "        self.fc1 = nn.Linear(3, 50)  # 3 input features\n",
    "        self.fc2 = nn.Linear(50, 1)  # Output 1 value (EOC accuracy)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f856b432-03c5-43bd-82a8-7b78c0e8f093",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate the model\n",
    "model = RegressionModel()\n",
    "\n",
    "# Step 3: Training\n",
    "criterion = nn.MSELoss()  # Mean Squared Error Loss\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)  # Adam optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2ab4d5fb-dcc8-46a5-9b23-45c81226441f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Loss: 0.5349588990211487\n",
      "Epoch 101, Loss: 0.0530269518494606\n",
      "Epoch 201, Loss: 0.034562185406684875\n",
      "Epoch 301, Loss: 0.029647935181856155\n",
      "Epoch 401, Loss: 0.02824144996702671\n",
      "Epoch 501, Loss: 0.02759462781250477\n",
      "Epoch 601, Loss: 0.027220698073506355\n",
      "Epoch 701, Loss: 0.02695312909781933\n",
      "Epoch 801, Loss: 0.02673872746527195\n",
      "Epoch 901, Loss: 0.02654481679201126\n"
     ]
    }
   ],
   "source": [
    "# Training loop\n",
    "for epoch in range(1000):\n",
    "    optimizer.zero_grad()\n",
    "    outputs = model(X_train_tensor)\n",
    "    loss = criterion(outputs.squeeze(), y_train_tensor)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    if epoch % 100 == 0:\n",
    "        print(f'Epoch {epoch+1}, Loss: {loss.item()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5df83a37-6248-46e1-960f-f7fbc3285f7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 0.027308469638228416\n"
     ]
    }
   ],
   "source": [
    "# Step 4: Evaluation\n",
    "model.eval()  # Set the model to evaluation mode\n",
    "with torch.no_grad():  # Turn off gradients for validation\n",
    "    predicted = model(X_test_tensor)\n",
    "    loss = criterion(predicted.squeeze(), y_test_tensor)\n",
    "print(f'Test Loss: {loss.item()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cf59bdcd-36cc-42b7-8a21-2e0ce12d6251",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 4.5: Evaluate on new_df\n",
    "\n",
    "#reassign columns of new_df\n",
    "columns = list(new_df.columns)\n",
    "columns.remove('EOC')\n",
    "columns.append('EOC')\n",
    "\n",
    "# Reassign the DataFrame to follow the new column order\n",
    "new_df = new_df[columns]\n",
    "#new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8ac00070-a41c-433b-8db7-84ee4d7e8492",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss 2: 0.05086226388812065\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/edmondniu/anaconda3/lib/python3.11/site-packages/torch/nn/modules/loss.py:535: UserWarning: Using a target size (torch.Size([1237, 1])) that is different to the input size (torch.Size([1237])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
      "  return F.mse_loss(input, target, reduction=self.reduction)\n"
     ]
    }
   ],
   "source": [
    "# Step 5: Prediction\n",
    "# Let's say you have new data in 'new_data' DataFrame\n",
    "new_X = new_df[['num_passed', 'engaged', 'length']].values\n",
    "new_X = scaler.transform(new_X)  # Normalize the new data\n",
    "new_X_tensor = torch.tensor(new_X, dtype=torch.float32)\n",
    "\n",
    "new_Y = new_df[['EOC']].values\n",
    "new_Y_tensor = torch.tensor(new_Y, dtype=torch.float32)\n",
    "\n",
    "model.eval()  # Make sure the model is in eval mode\n",
    "with torch.no_grad():  # Turn off gradients for prediction\n",
    "    predictions = model(new_X_tensor)\n",
    "    loss2 = criterion(predictions.squeeze(), new_Y_tensor)\n",
    "print(f'Test Loss 2: {loss2.item()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "31569c3e-0d7c-4c2e-b61f-b6305bcdd59d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>student_id</th>\n",
       "      <th>chapter_number</th>\n",
       "      <th>num_passed</th>\n",
       "      <th>engaged</th>\n",
       "      <th>length</th>\n",
       "      <th>EOC</th>\n",
       "      <th>Pred_EOC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8001</td>\n",
       "      <td>dcac4b3f-adfe-4ce6-be41-163e05f7703e</td>\n",
       "      <td>2.0</td>\n",
       "      <td>26</td>\n",
       "      <td>8646928.0</td>\n",
       "      <td>98</td>\n",
       "      <td>0.950617</td>\n",
       "      <td>0.737392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8002</td>\n",
       "      <td>dcac4b3f-adfe-4ce6-be41-163e05f7703e</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29</td>\n",
       "      <td>11188197.0</td>\n",
       "      <td>99</td>\n",
       "      <td>0.938776</td>\n",
       "      <td>0.756085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8003</td>\n",
       "      <td>dcac4b3f-adfe-4ce6-be41-163e05f7703e</td>\n",
       "      <td>5.0</td>\n",
       "      <td>12</td>\n",
       "      <td>9955761.0</td>\n",
       "      <td>84</td>\n",
       "      <td>0.868852</td>\n",
       "      <td>0.729665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8004</td>\n",
       "      <td>dcac4b3f-adfe-4ce6-be41-163e05f7703e</td>\n",
       "      <td>6.0</td>\n",
       "      <td>19</td>\n",
       "      <td>10342172.0</td>\n",
       "      <td>75</td>\n",
       "      <td>0.907216</td>\n",
       "      <td>0.672347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8005</td>\n",
       "      <td>dcac4b3f-adfe-4ce6-be41-163e05f7703e</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13</td>\n",
       "      <td>3481849.0</td>\n",
       "      <td>64</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>0.535543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1232</th>\n",
       "      <td>9233</td>\n",
       "      <td>ff6ec9fe-de1d-4b45-8136-59465d9c85ab</td>\n",
       "      <td>5.0</td>\n",
       "      <td>12</td>\n",
       "      <td>9989833.0</td>\n",
       "      <td>48</td>\n",
       "      <td>0.641791</td>\n",
       "      <td>0.665848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1233</th>\n",
       "      <td>9234</td>\n",
       "      <td>ff6ec9fe-de1d-4b45-8136-59465d9c85ab</td>\n",
       "      <td>6.0</td>\n",
       "      <td>19</td>\n",
       "      <td>16650582.0</td>\n",
       "      <td>56</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.664800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1234</th>\n",
       "      <td>9235</td>\n",
       "      <td>ff6ec9fe-de1d-4b45-8136-59465d9c85ab</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13</td>\n",
       "      <td>14206097.0</td>\n",
       "      <td>91</td>\n",
       "      <td>0.611650</td>\n",
       "      <td>0.697592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1235</th>\n",
       "      <td>9236</td>\n",
       "      <td>ff6ec9fe-de1d-4b45-8136-59465d9c85ab</td>\n",
       "      <td>10.0</td>\n",
       "      <td>12</td>\n",
       "      <td>8793743.0</td>\n",
       "      <td>40</td>\n",
       "      <td>0.469697</td>\n",
       "      <td>0.636404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1236</th>\n",
       "      <td>9237</td>\n",
       "      <td>ff6ec9fe-de1d-4b45-8136-59465d9c85ab</td>\n",
       "      <td>11.0</td>\n",
       "      <td>16</td>\n",
       "      <td>10646661.0</td>\n",
       "      <td>54</td>\n",
       "      <td>0.518519</td>\n",
       "      <td>0.665978</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1237 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                            student_id  chapter_number  \\\n",
       "0           8001  dcac4b3f-adfe-4ce6-be41-163e05f7703e             2.0   \n",
       "1           8002  dcac4b3f-adfe-4ce6-be41-163e05f7703e             3.0   \n",
       "2           8003  dcac4b3f-adfe-4ce6-be41-163e05f7703e             5.0   \n",
       "3           8004  dcac4b3f-adfe-4ce6-be41-163e05f7703e             6.0   \n",
       "4           8005  dcac4b3f-adfe-4ce6-be41-163e05f7703e             9.0   \n",
       "...          ...                                   ...             ...   \n",
       "1232        9233  ff6ec9fe-de1d-4b45-8136-59465d9c85ab             5.0   \n",
       "1233        9234  ff6ec9fe-de1d-4b45-8136-59465d9c85ab             6.0   \n",
       "1234        9235  ff6ec9fe-de1d-4b45-8136-59465d9c85ab             9.0   \n",
       "1235        9236  ff6ec9fe-de1d-4b45-8136-59465d9c85ab            10.0   \n",
       "1236        9237  ff6ec9fe-de1d-4b45-8136-59465d9c85ab            11.0   \n",
       "\n",
       "      num_passed     engaged  length       EOC  Pred_EOC  \n",
       "0             26   8646928.0      98  0.950617  0.737392  \n",
       "1             29  11188197.0      99  0.938776  0.756085  \n",
       "2             12   9955761.0      84  0.868852  0.729665  \n",
       "3             19  10342172.0      75  0.907216  0.672347  \n",
       "4             13   3481849.0      64  0.760000  0.535543  \n",
       "...          ...         ...     ...       ...       ...  \n",
       "1232          12   9989833.0      48  0.641791  0.665848  \n",
       "1233          19  16650582.0      56  0.500000  0.664800  \n",
       "1234          13  14206097.0      91  0.611650  0.697592  \n",
       "1235          12   8793743.0      40  0.469697  0.636404  \n",
       "1236          16  10646661.0      54  0.518519  0.665978  \n",
       "\n",
       "[1237 rows x 8 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Visualize the predicted vs actual values\n",
    "\n",
    "# Convert the tensor to a NumPy array and then to a pandas Series\n",
    "np_preds = pd.Series(predictions[:,0].numpy())\n",
    "new_df['Pred_EOC'] = np_preds\n",
    "\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9868a4fa-a020-4e10-ba98-df65da5c285a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#calculate the percentage error: (predicted - actual) / actual\n",
    "new_df_no0s = new_df[new_df['EOC'] != 0]\n",
    "\n",
    "new_df_no0s['perc_error'] = (   abs(new_df_no0s['Pred_EOC'] - new_df_no0s['EOC'])   ) / new_df_no0s['EOC']\n",
    "df = new_df_no0s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0ef612fe-3b0b-49ee-a209-c499266d64cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.24752948805150898"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['perc_error'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3fe8aad-78d8-4bbc-8300-5ef6f0e200ab",
   "metadata": {},
   "source": [
    "### Future improvements\n",
    "1. Tune Hyperparameters (k fold cross validation)\n",
    "\n",
    "For each combination of hyperparameters:\n",
    "\n",
    "Split your dataset into k folds.\n",
    "For each fold:\n",
    "Set aside one fold as the validation set, and combine the remaining k-1 folds into the training set.\n",
    "Train the model on the training set.\n",
    "Evaluate the model on the validation set.\n",
    "Calculate the average performance across all k folds.\n",
    "   \n",
    "3. More input dimensions\n",
    "4. Different model (that is better for predicting this kind of data, not a CNN)\n",
    "\n",
    "If you want a model that is robust, can handle non-linear patterns, and you have enough data, Random Forests might be a good starting point.\n",
    "If you believe the relationship in the data is highly non-linear and complex and you have the computational resources to fine-tune the model, an SVM with a non-linear kernel might be appropriate.\n",
    "If your dataset is not too large, the features have meaningful proximity relations, and computational efficiency is not a concern, you could consider KNN Regression.\n",
    "\n",
    "It's generally a good idea to try multiple models and use cross-validation to compare their performance based on a metric like RMSE (Root Mean Square Error) or MAE (Mean Absolute Error). The model that shows the best performance on a validation set (or via cross-validation) is often chosen as the best model for that specific dataset and task.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "df31bfbd-f4d7-459d-86f3-25aa5229c9c6",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 20\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epochs \u001b[38;5;129;01min\u001b[39;00m num_epochs_options:\n\u001b[1;32m     19\u001b[0m     validation_scores \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 20\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m train_index, val_index \u001b[38;5;129;01min\u001b[39;00m kf\u001b[38;5;241m.\u001b[39msplit(\u001b[43mX\u001b[49m):\n\u001b[1;32m     21\u001b[0m         \u001b[38;5;66;03m# Split data\u001b[39;00m\n\u001b[1;32m     22\u001b[0m         X_train_fold, X_val_fold \u001b[38;5;241m=\u001b[39m X[train_index], X[val_index]\n\u001b[1;32m     23\u001b[0m         y_train_fold, y_val_fold \u001b[38;5;241m=\u001b[39m y[train_index], y[val_index]\n",
      "\u001b[0;31mNameError\u001b[0m: name 'X' is not defined"
     ]
    }
   ],
   "source": [
    "#Cross Validation ChatGPT code\n",
    "import numpy as np\n",
    "from sklearn.model_selection import KFold\n",
    "from torch.utils.data import Subset, DataLoader\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "# Assuming X and y are your features and labels as numpy arrays\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "# Define hyperparameter grid (this is a simple example)\n",
    "learning_rates = [0.001, 0.01, 0.1]\n",
    "num_epochs_options = [100, 200]\n",
    "\n",
    "best_hyperparams = None\n",
    "best_validation_score = float('inf')\n",
    "\n",
    "for lr in learning_rates:\n",
    "    for epochs in num_epochs_options:\n",
    "        validation_scores = []\n",
    "        for train_index, val_index in kf.split(X):\n",
    "            # Split data\n",
    "            X_train_fold, X_val_fold = X[train_index], X[val_index]\n",
    "            y_train_fold, y_val_fold = y[train_index], y[val_index]\n",
    "\n",
    "            # Convert to PyTorch datasets, then to DataLoader if necessary\n",
    "            # For simplicity, this part is omitted, but you would convert\n",
    "            # the subsets to DataLoader objects here.\n",
    "\n",
    "            # Initialize model, criterion, and optimizer with current hyperparams\n",
    "            model = RegressionModel()  # Your model\n",
    "            criterion = nn.MSELoss()\n",
    "            optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "            # Train the model on this fold\n",
    "            # (Training loop is omitted for brevity)\n",
    "\n",
    "            # Evaluate the model on the validation fold\n",
    "            # (Evaluation code is omitted for brevity)\n",
    "            \n",
    "            # Compute validation score - e.g., mean squared error\n",
    "            validation_score = mean_squared_error(y_val_true, y_val_pred)\n",
    "            validation_scores.append(validation_score)\n",
    "\n",
    "        # Compute the average validation score for this hyperparameter combination\n",
    "        avg_validation_score = np.mean(validation_scores)\n",
    "        if avg_validation_score < best_validation_score:\n",
    "            best_validation_score = avg_validation_score\n",
    "            best_hyperparams = {'lr': lr, 'epochs': epochs}\n",
    "\n",
    "print(f\"Best Hyperparameters: {best_hyperparams}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}