dataset_selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Discover data features in **MachineLearningCVE**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "machineLearningCVE = pd.read_csv(\"/Users/Safa BENABDESSADOK/Desktop/M1 GL/SAD/SAD project/Dataset/CIC-IDS2017/MachineLearningCVE/1- Monday-WorkingHours.pcap_ISCX.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 529918 entries, 0 to 529917\n",
      "Data columns (total 79 columns):\n",
      " #   Column                        Non-Null Count   Dtype  \n",
      "---  ------                        --------------   -----  \n",
      " 0    Destination Port             529918 non-null  int64  \n",
      " 1    Flow Duration                529918 non-null  int64  \n",
      " 2    Total Fwd Packets            529918 non-null  int64  \n",
      " 3    Total Backward Packets       529918 non-null  int64  \n",
      " 4   Total Length of Fwd Packets   529918 non-null  int64  \n",
      " 5    Total Length of Bwd Packets  529918 non-null  int64  \n",
      " 6    Fwd Packet Length Max        529918 non-null  int64  \n",
      " 7    Fwd Packet Length Min        529918 non-null  int64  \n",
      " 8    Fwd Packet Length Mean       529918 non-null  float64\n",
      " 9    Fwd Packet Length Std        529918 non-null  float64\n",
      " 10  Bwd Packet Length Max         529918 non-null  int64  \n",
      " 11   Bwd Packet Length Min        529918 non-null  int64  \n",
      " 12   Bwd Packet Length Mean       529918 non-null  float64\n",
      " 13   Bwd Packet Length Std        529918 non-null  float64\n",
      " 14  Flow Bytes/s                  529854 non-null  float64\n",
      " 15   Flow Packets/s               529918 non-null  float64\n",
      " 16   Flow IAT Mean                529918 non-null  float64\n",
      " 17   Flow IAT Std                 529918 non-null  float64\n",
      " 18   Flow IAT Max                 529918 non-null  int64  \n",
      " 19   Flow IAT Min                 529918 non-null  int64  \n",
      " 20  Fwd IAT Total                 529918 non-null  int64  \n",
      " 21   Fwd IAT Mean                 529918 non-null  float64\n",
      " 22   Fwd IAT Std                  529918 non-null  float64\n",
      " 23   Fwd IAT Max                  529918 non-null  int64  \n",
      " 24   Fwd IAT Min                  529918 non-null  int64  \n",
      " 25  Bwd IAT Total                 529918 non-null  int64  \n",
      " 26   Bwd IAT Mean                 529918 non-null  float64\n",
      " 27   Bwd IAT Std                  529918 non-null  float64\n",
      " 28   Bwd IAT Max                  529918 non-null  int64  \n",
      " 29   Bwd IAT Min                  529918 non-null  int64  \n",
      " 30  Fwd PSH Flags                 529918 non-null  int64  \n",
      " 31   Bwd PSH Flags                529918 non-null  int64  \n",
      " 32   Fwd URG Flags                529918 non-null  int64  \n",
      " 33   Bwd URG Flags                529918 non-null  int64  \n",
      " 34   Fwd Header Length            529918 non-null  int64  \n",
      " 35   Bwd Header Length            529918 non-null  int64  \n",
      " 36  Fwd Packets/s                 529918 non-null  float64\n",
      " 37   Bwd Packets/s                529918 non-null  float64\n",
      " 38   Min Packet Length            529918 non-null  int64  \n",
      " 39   Max Packet Length            529918 non-null  int64  \n",
      " 40   Packet Length Mean           529918 non-null  float64\n",
      " 41   Packet Length Std            529918 non-null  float64\n",
      " 42   Packet Length Variance       529918 non-null  float64\n",
      " 43  FIN Flag Count                529918 non-null  int64  \n",
      " 44   SYN Flag Count               529918 non-null  int64  \n",
      " 45   RST Flag Count               529918 non-null  int64  \n",
      " 46   PSH Flag Count               529918 non-null  int64  \n",
      " 47   ACK Flag Count               529918 non-null  int64  \n",
      " 48   URG Flag Count               529918 non-null  int64  \n",
      " 49   CWE Flag Count               529918 non-null  int64  \n",
      " 50   ECE Flag Count               529918 non-null  int64  \n",
      " 51   Down/Up Ratio                529918 non-null  int64  \n",
      " 52   Average Packet Size          529918 non-null  float64\n",
      " 53   Avg Fwd Segment Size         529918 non-null  float64\n",
      " 54   Avg Bwd Segment Size         529918 non-null  float64\n",
      " 55   Fwd Header Length.1          529918 non-null  int64  \n",
      " 56  Fwd Avg Bytes/Bulk            529918 non-null  int64  \n",
      " 57   Fwd Avg Packets/Bulk         529918 non-null  int64  \n",
      " 58   Fwd Avg Bulk Rate            529918 non-null  int64  \n",
      " 59   Bwd Avg Bytes/Bulk           529918 non-null  int64  \n",
      " 60   Bwd Avg Packets/Bulk         529918 non-null  int64  \n",
      " 61  Bwd Avg Bulk Rate             529918 non-null  int64  \n",
      " 62  Subflow Fwd Packets           529918 non-null  int64  \n",
      " 63   Subflow Fwd Bytes            529918 non-null  int64  \n",
      " 64   Subflow Bwd Packets          529918 non-null  int64  \n",
      " 65   Subflow Bwd Bytes            529918 non-null  int64  \n",
      " 66  Init_Win_bytes_forward        529918 non-null  int64  \n",
      " 67   Init_Win_bytes_backward      529918 non-null  int64  \n",
      " 68   act_data_pkt_fwd             529918 non-null  int64  \n",
      " 69   min_seg_size_forward         529918 non-null  int64  \n",
      " 70  Active Mean                   529918 non-null  float64\n",
      " 71   Active Std                   529918 non-null  float64\n",
      " 72   Active Max                   529918 non-null  int64  \n",
      " 73   Active Min                   529918 non-null  int64  \n",
      " 74  Idle Mean                     529918 non-null  float64\n",
      " 75   Idle Std                     529918 non-null  float64\n",
      " 76   Idle Max                     529918 non-null  int64  \n",
      " 77   Idle Min                     529918 non-null  int64  \n",
      " 78   Label                        529918 non-null  object \n",
      "dtypes: float64(24), int64(54), object(1)\n",
      "memory usage: 319.4+ MB\n"
     ]
    }
   ],
   "source": [
    "machineLearningCVE.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Destination Port</th>\n",
       "      <th>Flow Duration</th>\n",
       "      <th>Total Fwd Packets</th>\n",
       "      <th>Total Backward Packets</th>\n",
       "      <th>Total Length of Fwd Packets</th>\n",
       "      <th>Total Length of Bwd Packets</th>\n",
       "      <th>Fwd Packet Length Max</th>\n",
       "      <th>Fwd Packet Length Min</th>\n",
       "      <th>Fwd Packet Length Mean</th>\n",
       "      <th>Fwd Packet Length Std</th>\n",
       "      <th>Bwd Packet Length Max</th>\n",
       "      <th>Bwd Packet Length Min</th>\n",
       "      <th>Bwd Packet Length Mean</th>\n",
       "      <th>Bwd Packet Length Std</th>\n",
       "      <th>Flow Bytes/s</th>\n",
       "      <th>Flow Packets/s</th>\n",
       "      <th>Flow IAT Mean</th>\n",
       "      <th>Flow IAT Std</th>\n",
       "      <th>Flow IAT Max</th>\n",
       "      <th>Flow IAT Min</th>\n",
       "      <th>Fwd IAT Total</th>\n",
       "      <th>Fwd IAT Mean</th>\n",
       "      <th>Fwd IAT Std</th>\n",
       "      <th>Fwd IAT Max</th>\n",
       "      <th>Fwd IAT Min</th>\n",
       "      <th>Bwd IAT Total</th>\n",
       "      <th>Bwd IAT Mean</th>\n",
       "      <th>Bwd IAT Std</th>\n",
       "      <th>Bwd IAT Max</th>\n",
       "      <th>Bwd IAT Min</th>\n",
       "      <th>Fwd PSH Flags</th>\n",
       "      <th>Bwd PSH Flags</th>\n",
       "      <th>Fwd URG Flags</th>\n",
       "      <th>Bwd URG Flags</th>\n",
       "      <th>Fwd Header Length</th>\n",
       "      <th>Bwd Header Length</th>\n",
       "      <th>Fwd Packets/s</th>\n",
       "      <th>Bwd Packets/s</th>\n",
       "      <th>Min Packet Length</th>\n",
       "      <th>Max Packet Length</th>\n",
       "      <th>Packet Length Mean</th>\n",
       "      <th>Packet Length Std</th>\n",
       "      <th>Packet Length Variance</th>\n",
       "      <th>FIN Flag Count</th>\n",
       "      <th>SYN Flag Count</th>\n",
       "      <th>RST Flag Count</th>\n",
       "      <th>PSH Flag Count</th>\n",
       "      <th>ACK Flag Count</th>\n",
       "      <th>URG Flag Count</th>\n",
       "      <th>CWE Flag Count</th>\n",
       "      <th>ECE Flag Count</th>\n",
       "      <th>Down/Up Ratio</th>\n",
       "      <th>Average Packet Size</th>\n",
       "      <th>Avg Fwd Segment Size</th>\n",
       "      <th>Avg Bwd Segment Size</th>\n",
       "      <th>Fwd Header Length.1</th>\n",
       "      <th>Fwd Avg Bytes/Bulk</th>\n",
       "      <th>Fwd Avg Packets/Bulk</th>\n",
       "      <th>Fwd Avg Bulk Rate</th>\n",
       "      <th>Bwd Avg Bytes/Bulk</th>\n",
       "      <th>Bwd Avg Packets/Bulk</th>\n",
       "      <th>Bwd Avg Bulk Rate</th>\n",
       "      <th>Subflow Fwd Packets</th>\n",
       "      <th>Subflow Fwd Bytes</th>\n",
       "      <th>Subflow Bwd Packets</th>\n",
       "      <th>Subflow Bwd Bytes</th>\n",
       "      <th>Init_Win_bytes_forward</th>\n",
       "      <th>Init_Win_bytes_backward</th>\n",
       "      <th>act_data_pkt_fwd</th>\n",
       "      <th>min_seg_size_forward</th>\n",
       "      <th>Active Mean</th>\n",
       "      <th>Active Std</th>\n",
       "      <th>Active Max</th>\n",
       "      <th>Active Min</th>\n",
       "      <th>Idle Mean</th>\n",
       "      <th>Idle Std</th>\n",
       "      <th>Idle Max</th>\n",
       "      <th>Idle Min</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49188</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3000000.0</td>\n",
       "      <td>500000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>500000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>329</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Destination Port   Flow Duration   Total Fwd Packets  \\\n",
       "0              49188               4                   2   \n",
       "\n",
       "    Total Backward Packets  Total Length of Fwd Packets  \\\n",
       "0                        0                           12   \n",
       "\n",
       "    Total Length of Bwd Packets   Fwd Packet Length Max  \\\n",
       "0                             0                       6   \n",
       "\n",
       "    Fwd Packet Length Min   Fwd Packet Length Mean   Fwd Packet Length Std  \\\n",
       "0                       6                      6.0                     0.0   \n",
       "\n",
       "   Bwd Packet Length Max   Bwd Packet Length Min   Bwd Packet Length Mean  \\\n",
       "0                      0                       0                      0.0   \n",
       "\n",
       "    Bwd Packet Length Std  Flow Bytes/s   Flow Packets/s   Flow IAT Mean  \\\n",
       "0                     0.0     3000000.0         500000.0             4.0   \n",
       "\n",
       "    Flow IAT Std   Flow IAT Max   Flow IAT Min  Fwd IAT Total   Fwd IAT Mean  \\\n",
       "0            0.0              4              4              4            4.0   \n",
       "\n",
       "    Fwd IAT Std   Fwd IAT Max   Fwd IAT Min  Bwd IAT Total   Bwd IAT Mean  \\\n",
       "0           0.0             4             4              0            0.0   \n",
       "\n",
       "    Bwd IAT Std   Bwd IAT Max   Bwd IAT Min  Fwd PSH Flags   Bwd PSH Flags  \\\n",
       "0           0.0             0             0              0               0   \n",
       "\n",
       "    Fwd URG Flags   Bwd URG Flags   Fwd Header Length   Bwd Header Length  \\\n",
       "0               0               0                  40                   0   \n",
       "\n",
       "   Fwd Packets/s   Bwd Packets/s   Min Packet Length   Max Packet Length  \\\n",
       "0       500000.0             0.0                   6                   6   \n",
       "\n",
       "    Packet Length Mean   Packet Length Std   Packet Length Variance  \\\n",
       "0                  6.0                 0.0                      0.0   \n",
       "\n",
       "   FIN Flag Count   SYN Flag Count   RST Flag Count   PSH Flag Count  \\\n",
       "0               0                0                0                0   \n",
       "\n",
       "    ACK Flag Count   URG Flag Count   CWE Flag Count   ECE Flag Count  \\\n",
       "0                1                1                0                0   \n",
       "\n",
       "    Down/Up Ratio   Average Packet Size   Avg Fwd Segment Size  \\\n",
       "0               0                   9.0                    6.0   \n",
       "\n",
       "    Avg Bwd Segment Size   Fwd Header Length.1  Fwd Avg Bytes/Bulk  \\\n",
       "0                    0.0                    40                   0   \n",
       "\n",
       "    Fwd Avg Packets/Bulk   Fwd Avg Bulk Rate   Bwd Avg Bytes/Bulk  \\\n",
       "0                      0                   0                    0   \n",
       "\n",
       "    Bwd Avg Packets/Bulk  Bwd Avg Bulk Rate  Subflow Fwd Packets  \\\n",
       "0                      0                  0                    2   \n",
       "\n",
       "    Subflow Fwd Bytes   Subflow Bwd Packets   Subflow Bwd Bytes  \\\n",
       "0                  12                     0                   0   \n",
       "\n",
       "   Init_Win_bytes_forward   Init_Win_bytes_backward   act_data_pkt_fwd  \\\n",
       "0                     329                        -1                  1   \n",
       "\n",
       "    min_seg_size_forward  Active Mean   Active Std   Active Max   Active Min  \\\n",
       "0                     20          0.0          0.0            0            0   \n",
       "\n",
       "   Idle Mean   Idle Std   Idle Max   Idle Min   Label  \n",
       "0        0.0        0.0          0          0  BENIGN  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "machineLearningCVE.head(1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Discover data features in **TrafficLabelling**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "trafficLabelling = pd.read_csv(\"/Users/Safa BENABDESSADOK/Desktop/M1 GL/SAD/SAD project/Dataset/CIC-IDS2017/TrafficLabelling/1- Monday-WorkingHours.pcap_ISCX.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 529918 entries, 0 to 529917\n",
      "Data columns (total 85 columns):\n",
      " #   Column                        Non-Null Count   Dtype  \n",
      "---  ------                        --------------   -----  \n",
      " 0   Flow ID                       529918 non-null  object \n",
      " 1    Source IP                    529918 non-null  object \n",
      " 2    Source Port                  529918 non-null  int64  \n",
      " 3    Destination IP               529918 non-null  object \n",
      " 4    Destination Port             529918 non-null  int64  \n",
      " 5    Protocol                     529918 non-null  int64  \n",
      " 6    Timestamp                    529918 non-null  object \n",
      " 7    Flow Duration                529918 non-null  int64  \n",
      " 8    Total Fwd Packets            529918 non-null  int64  \n",
      " 9    Total Backward Packets       529918 non-null  int64  \n",
      " 10  Total Length of Fwd Packets   529918 non-null  float64\n",
      " 11   Total Length of Bwd Packets  529918 non-null  float64\n",
      " 12   Fwd Packet Length Max        529918 non-null  float64\n",
      " 13   Fwd Packet Length Min        529918 non-null  float64\n",
      " 14   Fwd Packet Length Mean       529918 non-null  float64\n",
      " 15   Fwd Packet Length Std        529918 non-null  float64\n",
      " 16  Bwd Packet Length Max         529918 non-null  float64\n",
      " 17   Bwd Packet Length Min        529918 non-null  float64\n",
      " 18   Bwd Packet Length Mean       529918 non-null  float64\n",
      " 19   Bwd Packet Length Std        529918 non-null  float64\n",
      " 20  Flow Bytes/s                  529854 non-null  float64\n",
      " 21   Flow Packets/s               529918 non-null  float64\n",
      " 22   Flow IAT Mean                529918 non-null  float64\n",
      " 23   Flow IAT Std                 529918 non-null  float64\n",
      " 24   Flow IAT Max                 529918 non-null  float64\n",
      " 25   Flow IAT Min                 529918 non-null  float64\n",
      " 26  Fwd IAT Total                 529918 non-null  float64\n",
      " 27   Fwd IAT Mean                 529918 non-null  float64\n",
      " 28   Fwd IAT Std                  529918 non-null  float64\n",
      " 29   Fwd IAT Max                  529918 non-null  float64\n",
      " 30   Fwd IAT Min                  529918 non-null  float64\n",
      " 31  Bwd IAT Total                 529918 non-null  float64\n",
      " 32   Bwd IAT Mean                 529918 non-null  float64\n",
      " 33   Bwd IAT Std                  529918 non-null  float64\n",
      " 34   Bwd IAT Max                  529918 non-null  float64\n",
      " 35   Bwd IAT Min                  529918 non-null  float64\n",
      " 36  Fwd PSH Flags                 529918 non-null  int64  \n",
      " 37   Bwd PSH Flags                529918 non-null  int64  \n",
      " 38   Fwd URG Flags                529918 non-null  int64  \n",
      " 39   Bwd URG Flags                529918 non-null  int64  \n",
      " 40   Fwd Header Length            529918 non-null  int64  \n",
      " 41   Bwd Header Length            529918 non-null  int64  \n",
      " 42  Fwd Packets/s                 529918 non-null  float64\n",
      " 43   Bwd Packets/s                529918 non-null  float64\n",
      " 44   Min Packet Length            529918 non-null  float64\n",
      " 45   Max Packet Length            529918 non-null  float64\n",
      " 46   Packet Length Mean           529918 non-null  float64\n",
      " 47   Packet Length Std            529918 non-null  float64\n",
      " 48   Packet Length Variance       529918 non-null  float64\n",
      " 49  FIN Flag Count                529918 non-null  int64  \n",
      " 50   SYN Flag Count               529918 non-null  int64  \n",
      " 51   RST Flag Count               529918 non-null  int64  \n",
      " 52   PSH Flag Count               529918 non-null  int64  \n",
      " 53   ACK Flag Count               529918 non-null  int64  \n",
      " 54   URG Flag Count               529918 non-null  int64  \n",
      " 55   CWE Flag Count               529918 non-null  int64  \n",
      " 56   ECE Flag Count               529918 non-null  int64  \n",
      " 57   Down/Up Ratio                529918 non-null  float64\n",
      " 58   Average Packet Size          529918 non-null  float64\n",
      " 59   Avg Fwd Segment Size         529918 non-null  float64\n",
      " 60   Avg Bwd Segment Size         529918 non-null  float64\n",
      " 61   Fwd Header Length.1          529918 non-null  int64  \n",
      " 62  Fwd Avg Bytes/Bulk            529918 non-null  int64  \n",
      " 63   Fwd Avg Packets/Bulk         529918 non-null  int64  \n",
      " 64   Fwd Avg Bulk Rate            529918 non-null  int64  \n",
      " 65   Bwd Avg Bytes/Bulk           529918 non-null  int64  \n",
      " 66   Bwd Avg Packets/Bulk         529918 non-null  int64  \n",
      " 67  Bwd Avg Bulk Rate             529918 non-null  int64  \n",
      " 68  Subflow Fwd Packets           529918 non-null  int64  \n",
      " 69   Subflow Fwd Bytes            529918 non-null  int64  \n",
      " 70   Subflow Bwd Packets          529918 non-null  int64  \n",
      " 71   Subflow Bwd Bytes            529918 non-null  int64  \n",
      " 72  Init_Win_bytes_forward        529918 non-null  int64  \n",
      " 73   Init_Win_bytes_backward      529918 non-null  int64  \n",
      " 74   act_data_pkt_fwd             529918 non-null  int64  \n",
      " 75   min_seg_size_forward         529918 non-null  int64  \n",
      " 76  Active Mean                   529918 non-null  float64\n",
      " 77   Active Std                   529918 non-null  float64\n",
      " 78   Active Max                   529918 non-null  float64\n",
      " 79   Active Min                   529918 non-null  float64\n",
      " 80  Idle Mean                     529918 non-null  float64\n",
      " 81   Idle Std                     529918 non-null  float64\n",
      " 82   Idle Max                     529918 non-null  float64\n",
      " 83   Idle Min                     529918 non-null  float64\n",
      " 84   Label                        529918 non-null  object \n",
      "dtypes: float64(45), int64(35), object(5)\n",
      "memory usage: 343.7+ MB\n"
     ]
    }
   ],
   "source": [
    "trafficLabelling.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Flow ID</th>\n",
       "      <th>Source IP</th>\n",
       "      <th>Source Port</th>\n",
       "      <th>Destination IP</th>\n",
       "      <th>Destination Port</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Flow Duration</th>\n",
       "      <th>Total Fwd Packets</th>\n",
       "      <th>Total Backward Packets</th>\n",
       "      <th>Total Length of Fwd Packets</th>\n",
       "      <th>Total Length of Bwd Packets</th>\n",
       "      <th>Fwd Packet Length Max</th>\n",
       "      <th>Fwd Packet Length Min</th>\n",
       "      <th>Fwd Packet Length Mean</th>\n",
       "      <th>Fwd Packet Length Std</th>\n",
       "      <th>Bwd Packet Length Max</th>\n",
       "      <th>Bwd Packet Length Min</th>\n",
       "      <th>Bwd Packet Length Mean</th>\n",
       "      <th>Bwd Packet Length Std</th>\n",
       "      <th>Flow Bytes/s</th>\n",
       "      <th>Flow Packets/s</th>\n",
       "      <th>Flow IAT Mean</th>\n",
       "      <th>Flow IAT Std</th>\n",
       "      <th>Flow IAT Max</th>\n",
       "      <th>Flow IAT Min</th>\n",
       "      <th>Fwd IAT Total</th>\n",
       "      <th>Fwd IAT Mean</th>\n",
       "      <th>Fwd IAT Std</th>\n",
       "      <th>Fwd IAT Max</th>\n",
       "      <th>Fwd IAT Min</th>\n",
       "      <th>Bwd IAT Total</th>\n",
       "      <th>Bwd IAT Mean</th>\n",
       "      <th>Bwd IAT Std</th>\n",
       "      <th>Bwd IAT Max</th>\n",
       "      <th>Bwd IAT Min</th>\n",
       "      <th>Fwd PSH Flags</th>\n",
       "      <th>Bwd PSH Flags</th>\n",
       "      <th>Fwd URG Flags</th>\n",
       "      <th>Bwd URG Flags</th>\n",
       "      <th>Fwd Header Length</th>\n",
       "      <th>Bwd Header Length</th>\n",
       "      <th>Fwd Packets/s</th>\n",
       "      <th>Bwd Packets/s</th>\n",
       "      <th>Min Packet Length</th>\n",
       "      <th>Max Packet Length</th>\n",
       "      <th>Packet Length Mean</th>\n",
       "      <th>Packet Length Std</th>\n",
       "      <th>Packet Length Variance</th>\n",
       "      <th>FIN Flag Count</th>\n",
       "      <th>SYN Flag Count</th>\n",
       "      <th>RST Flag Count</th>\n",
       "      <th>PSH Flag Count</th>\n",
       "      <th>ACK Flag Count</th>\n",
       "      <th>URG Flag Count</th>\n",
       "      <th>CWE Flag Count</th>\n",
       "      <th>ECE Flag Count</th>\n",
       "      <th>Down/Up Ratio</th>\n",
       "      <th>Average Packet Size</th>\n",
       "      <th>Avg Fwd Segment Size</th>\n",
       "      <th>Avg Bwd Segment Size</th>\n",
       "      <th>Fwd Header Length.1</th>\n",
       "      <th>Fwd Avg Bytes/Bulk</th>\n",
       "      <th>Fwd Avg Packets/Bulk</th>\n",
       "      <th>Fwd Avg Bulk Rate</th>\n",
       "      <th>Bwd Avg Bytes/Bulk</th>\n",
       "      <th>Bwd Avg Packets/Bulk</th>\n",
       "      <th>Bwd Avg Bulk Rate</th>\n",
       "      <th>Subflow Fwd Packets</th>\n",
       "      <th>Subflow Fwd Bytes</th>\n",
       "      <th>Subflow Bwd Packets</th>\n",
       "      <th>Subflow Bwd Bytes</th>\n",
       "      <th>Init_Win_bytes_forward</th>\n",
       "      <th>Init_Win_bytes_backward</th>\n",
       "      <th>act_data_pkt_fwd</th>\n",
       "      <th>min_seg_size_forward</th>\n",
       "      <th>Active Mean</th>\n",
       "      <th>Active Std</th>\n",
       "      <th>Active Max</th>\n",
       "      <th>Active Min</th>\n",
       "      <th>Idle Mean</th>\n",
       "      <th>Idle Std</th>\n",
       "      <th>Idle Max</th>\n",
       "      <th>Idle Min</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>192.168.10.5-8.254.250.126-49188-80-6</td>\n",
       "      <td>8.254.250.126</td>\n",
       "      <td>80</td>\n",
       "      <td>192.168.10.5</td>\n",
       "      <td>49188</td>\n",
       "      <td>6</td>\n",
       "      <td>03/07/2017 08:55:58</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3000000.0</td>\n",
       "      <td>500000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>500000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>329</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Flow ID      Source IP   Source Port  \\\n",
       "0  192.168.10.5-8.254.250.126-49188-80-6  8.254.250.126            80   \n",
       "\n",
       "   Destination IP   Destination Port   Protocol            Timestamp  \\\n",
       "0    192.168.10.5              49188          6  03/07/2017 08:55:58   \n",
       "\n",
       "    Flow Duration   Total Fwd Packets   Total Backward Packets  \\\n",
       "0               4                   2                        0   \n",
       "\n",
       "   Total Length of Fwd Packets   Total Length of Bwd Packets  \\\n",
       "0                         12.0                           0.0   \n",
       "\n",
       "    Fwd Packet Length Max   Fwd Packet Length Min   Fwd Packet Length Mean  \\\n",
       "0                     6.0                     6.0                      6.0   \n",
       "\n",
       "    Fwd Packet Length Std  Bwd Packet Length Max   Bwd Packet Length Min  \\\n",
       "0                     0.0                    0.0                     0.0   \n",
       "\n",
       "    Bwd Packet Length Mean   Bwd Packet Length Std  Flow Bytes/s  \\\n",
       "0                      0.0                     0.0     3000000.0   \n",
       "\n",
       "    Flow Packets/s   Flow IAT Mean   Flow IAT Std   Flow IAT Max  \\\n",
       "0         500000.0             4.0            0.0            4.0   \n",
       "\n",
       "    Flow IAT Min  Fwd IAT Total   Fwd IAT Mean   Fwd IAT Std   Fwd IAT Max  \\\n",
       "0            4.0            4.0            4.0           0.0           4.0   \n",
       "\n",
       "    Fwd IAT Min  Bwd IAT Total   Bwd IAT Mean   Bwd IAT Std   Bwd IAT Max  \\\n",
       "0           4.0            0.0            0.0           0.0           0.0   \n",
       "\n",
       "    Bwd IAT Min  Fwd PSH Flags   Bwd PSH Flags   Fwd URG Flags  \\\n",
       "0           0.0              0               0               0   \n",
       "\n",
       "    Bwd URG Flags   Fwd Header Length   Bwd Header Length  Fwd Packets/s  \\\n",
       "0               0                  40                   0       500000.0   \n",
       "\n",
       "    Bwd Packets/s   Min Packet Length   Max Packet Length  \\\n",
       "0             0.0                 6.0                 6.0   \n",
       "\n",
       "    Packet Length Mean   Packet Length Std   Packet Length Variance  \\\n",
       "0                  6.0                 0.0                      0.0   \n",
       "\n",
       "   FIN Flag Count   SYN Flag Count   RST Flag Count   PSH Flag Count  \\\n",
       "0               0                0                0                0   \n",
       "\n",
       "    ACK Flag Count   URG Flag Count   CWE Flag Count   ECE Flag Count  \\\n",
       "0                1                1                0                0   \n",
       "\n",
       "    Down/Up Ratio   Average Packet Size   Avg Fwd Segment Size  \\\n",
       "0             0.0                   9.0                    6.0   \n",
       "\n",
       "    Avg Bwd Segment Size   Fwd Header Length.1  Fwd Avg Bytes/Bulk  \\\n",
       "0                    0.0                    40                   0   \n",
       "\n",
       "    Fwd Avg Packets/Bulk   Fwd Avg Bulk Rate   Bwd Avg Bytes/Bulk  \\\n",
       "0                      0                   0                    0   \n",
       "\n",
       "    Bwd Avg Packets/Bulk  Bwd Avg Bulk Rate  Subflow Fwd Packets  \\\n",
       "0                      0                  0                    2   \n",
       "\n",
       "    Subflow Fwd Bytes   Subflow Bwd Packets   Subflow Bwd Bytes  \\\n",
       "0                  12                     0                   0   \n",
       "\n",
       "   Init_Win_bytes_forward   Init_Win_bytes_backward   act_data_pkt_fwd  \\\n",
       "0                     329                        -1                  1   \n",
       "\n",
       "    min_seg_size_forward  Active Mean   Active Std   Active Max   Active Min  \\\n",
       "0                     20          0.0          0.0          0.0          0.0   \n",
       "\n",
       "   Idle Mean   Idle Std   Idle Max   Idle Min   Label  \n",
       "0        0.0        0.0        0.0        0.0  BENIGN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trafficLabelling.head(1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The additional features in trafficLabelling data files "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Flow ID\n",
      " Source IP\n",
      " Source Port\n",
      " Destination IP\n",
      " Protocol\n",
      " Timestamp\n"
     ]
    }
   ],
   "source": [
    "arr = np.in1d(trafficLabelling.columns.values, machineLearningCVE.columns.values)\n",
    "# arr is an array containing if a feature in 'traffic labeling' exist in 'machineLearningCVE' or not (boolean array).\n",
    "for i in range(arr.size):\n",
    "    if not arr[i] :\n",
    "        print(trafficLabelling.columns.values[i])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Those features \"Flow ID, Source IP, Source Port, Destination IP, Protocol, Timestamp\" are not good to be considered in a ML model because:\n",
    "- *Flow ID* contain repeated information which is *Destination IP*, *Source IP*, *Destination Port*, *Source Port*, and *Protocol* \n",
    "- *Source IP*, *Source Port*, and  *Destination IP* are changing continuously.\n",
    "- *Protocol* can be deduced from *Destination Port*"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## As a result of this, we choose to work on **MachineLearningCVE** data files"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12 (main, Apr  4 2022, 05:22:27) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a3394be68f6686ff919e9217acf1f3dfbb710015f7bd218178b014d900b478a9"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}