test_LODA_RS-Hash.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4c94ed90",
   "metadata": {},
   "source": [
    "# Test LODA and RS-Hash with River methods\n",
    "\n",
    "The objective of this notebook is to test LODA and RS-Hash methods adapted to a River framework (but not fitting perfectly the guidelines, and still using numpy arrays)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49c030be",
   "metadata": {},
   "source": [
    "### imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0030079f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39b058b3",
   "metadata": {},
   "source": [
    "Arrhythmia dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6a3f78cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.io\n",
    "#data = scipy.io.loadmat('/Users/Sophie/Library/CloudStorage/OneDrive-Personnel/IP/Data Stream Processing/PySAD_to_River/data/cover.mat') \n",
    "data_arr = scipy.io.loadmat('C:/Users/e32cl/Documents/M2/P2 Data Stream/Projet/projet_v2/PySAD_to_River/data/arrhythmia.mat') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8efcf1be",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data_arr\n",
    "X = data['X']\n",
    "y = data['y']\n",
    "y_flat = data['y'].flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a6b22d01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'> (452, 274)\n",
      "<class 'numpy.ndarray'> (452, 1)\n",
      "(452,)\n"
     ]
    }
   ],
   "source": [
    "print(type(X),X.shape)\n",
    "print(type(y),y.shape)\n",
    "print(y.flatten().shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ca1ca697",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14601769911504425"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc51b920",
   "metadata": {},
   "source": [
    "#### import our models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a5ae09ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "from anomaly.base import AnomalyDetector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "96600b38",
   "metadata": {},
   "outputs": [],
   "source": [
    "from river import utils, metrics\n",
    "from river.stream import iter_array, iter_pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4f96f26c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.loda import LODA\n",
    "from models.rs_hash import RSHash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "50406f5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pysad.evaluation import AUROCMetric\n",
    "from pysad.transform.probability_calibration import ConformalProbabilityCalibrator"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be7e58ea",
   "metadata": {},
   "source": [
    "### LODA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a51dcf92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "9.000133971466786e-14\n",
      "ROCAUC River :  ROCAUC: 50.00%\n",
      "ROCAUC sklearn :  0.49870466321243523\n"
     ]
    }
   ],
   "source": [
    "model = LODA()\n",
    "\n",
    "# River ROC-AUC\n",
    "ROCAUC = metrics.ROCAUC()\n",
    "# PySAD/sklearn ROC-AUC\n",
    "ROCAUC_1 = AUROCMetric()\n",
    "\n",
    "i = 0\n",
    "for xi, yi in iter_array(X, y_flat):\n",
    "    model = model.learn_one(xi)\n",
    "    anomaly_score = model.score_one(xi)\n",
    "    if i%10 ==0:\n",
    "        print(anomaly_score)\n",
    "    i+=1\n",
    "    ROCAUC.update(yi, anomaly_score)\n",
    "    ROCAUC_1.update(yi, anomaly_score)\n",
    "print('ROCAUC River : ', ROCAUC)\n",
    "print('ROCAUC sklearn : ', ROCAUC_1.get())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0e7b5b3",
   "metadata": {},
   "source": [
    "Anomaly scores go to 0 $(10^{-14})$, seems to be an issue to use histograms and projections using only one sample at a time.\n",
    "Same results obtained with PySAD, and a calibrator is not enough for this issue."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1680a419",
   "metadata": {},
   "source": [
    "### RS-Hash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ec95a8da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROCAUC River :  ROCAUC: -0.00%\n",
      "ROCAUC sklearn :  0.7524336630554247\n"
     ]
    }
   ],
   "source": [
    "model = RSHash(X.min(axis=0), X.max(axis=0))\n",
    "\n",
    "# River ROC-AUC\n",
    "ROCAUC = metrics.ROCAUC()\n",
    "# PySAD/sklearn ROC-AUC\n",
    "ROCAUC_1 = AUROCMetric()\n",
    "\n",
    "for xi, yi in iter_array(X, y):\n",
    "    model = model.learn_one(xi)          \n",
    "    anomaly_score = model.score_one(xi)\n",
    "    ROCAUC.update(yi[0], -anomaly_score)\n",
    "    ROCAUC_1.update(yi[0], -anomaly_score)\n",
    "print('ROCAUC River : ', ROCAUC)\n",
    "print('ROCAUC sklearn : ', ROCAUC_1.get())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5242ce19",
   "metadata": {},
   "source": [
    "RS-Hash : take as input the minimum and maximum boundaries of the features : how to implement this for online learning with one sample ? Potential solution : rolling preprocessing, but not easy to implement and maybe not relevant for this problem.\n",
    "Anomalousness score is reversed : low score = high potential of abnormality (use a minus in the ROC-AUC update ?)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f60d839",
   "metadata": {},
   "source": [
    "#### Test PySAD calibrator\n",
    "Test the Conformal Probability Calibrator, with a window size of 300 on RS-Hash implementation, to check if the results are improved/changed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ff69d0f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.]\n",
      "[0.18181818]\n",
      "[0.42857143]\n",
      "[0.]\n",
      "[0.04878049]\n",
      "[0.17647059]\n",
      "[0.42622951]\n",
      "[0.07042254]\n",
      "[0.]\n",
      "[0.15384615]\n",
      "[0.11881188]\n",
      "[0.09009009]\n",
      "[0.10743802]\n",
      "[0.00763359]\n",
      "[0.]\n",
      "[0.10596026]\n",
      "[0.26708075]\n",
      "[0.04678363]\n",
      "[0.]\n",
      "[0.17801047]\n",
      "[0.1641791]\n",
      "[0.71563981]\n",
      "[0.02714932]\n",
      "[0.10822511]\n",
      "[0.04149378]\n",
      "[0.60557769]\n",
      "[0.29501916]\n",
      "[0.]\n",
      "[0.70462633]\n",
      "[0.19931271]\n",
      "[0.65666667]\n",
      "[0.31333333]\n",
      "[0.65666667]\n",
      "[0.57]\n",
      "[0.18333333]\n",
      "[0.82333333]\n",
      "[0.10666667]\n",
      "[0.93333333]\n",
      "[0.51333333]\n",
      "[0.04333333]\n",
      "[0.25666667]\n",
      "[0.51]\n",
      "[0.42333333]\n",
      "[0.98]\n",
      "[0.56333333]\n",
      "[0.29]\n",
      "ROCAUC River :  ROCAUC: 68.51%\n",
      "ROCAUC sklearn :  0.6870976605432564\n"
     ]
    }
   ],
   "source": [
    "model = RSHash(X.min(axis=0), X.max(axis=0))\n",
    "\n",
    "calibrator = ConformalProbabilityCalibrator(windowed=True, window_size=300)\n",
    "# River ROC-AUC\n",
    "ROCAUC = metrics.ROCAUC()\n",
    "# PySAD/sklearn ROC-AUC\n",
    "ROCAUC_1 = AUROCMetric()\n",
    "\n",
    "i = 0\n",
    "for xi, yi in iter_array(X, y):\n",
    "    model = model.learn_one(xi)          \n",
    "    anomaly_score = model.score_one(xi)\n",
    "    calibrated_score = calibrator.fit_transform(np.array([anomaly_score]))\n",
    "    if i%10 == 0:\n",
    "        print(calibrated_score)\n",
    "    i+=1\n",
    "    ROCAUC.update(yi[0], calibrated_score[0])\n",
    "    ROCAUC_1.update(yi[0], calibrated_score[0])\n",
    "print('ROCAUC River : ', ROCAUC)\n",
    "print('ROCAUC sklearn : ', ROCAUC_1.get())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30c6a9ab",
   "metadata": {},
   "source": [
    "Calibration may work with River ROCAUC, sklearn ROCAUC already has a calibrator, maybe over-computation that reduces the performance ?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}