Summit2024_monitoring.ipynb

{
 "metadata": {
  "kernelspec": {
   "display_name": "hf_public",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat_minor": 2,
 "nbformat": 4,
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "name": "CELL2",
    "collapsed": false
   },
   "source": "# Model and Feature Data Drift in Snowflake\n",
   "id": "e31244e2-0c4a-4888-b13b-eee05b3ebe06"
  },
  {
   "cell_type": "code",
   "id": "7ce5c504-1693-49f2-9ad5-bce8503c2bb3",
   "metadata": {
    "language": "python",
    "name": "cell3",
    "collapsed": false
   },
   "outputs": [],
   "source": "!pip install -q snowflake-ml-python==1.5.0\n!pip install -q matplotlib\n!pip install -q seaborn\n!pip install -q evidently",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "name": "CELL3",
    "language": "python",
    "collapsed": false
   },
   "outputs": [],
   "source": "# Import python packages\n#import streamlit as st\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# We can also use Snowpark for our analyses!\nfrom snowflake.snowpark.context import get_active_session\nsession = get_active_session()",
   "id": "92d7475f-9a5a-4b52-b7c3-04d930c96a4b"
  },
  {
   "cell_type": "code",
   "id": "5dfd038d-462b-496c-ab91-cbf44c579dc8",
   "metadata": {
    "language": "python",
    "name": "cell8",
    "collapsed": false
   },
   "outputs": [],
   "source": "snowflake_environment = session.sql('select current_user(), current_version()').collect()\nfrom snowflake.snowpark.version import VERSION\nfrom snowflake.ml import version\n\n# Current Environment Details\nprint('User                        : {}'.format(snowflake_environment[0][0]))\nprint('Role                        : {}'.format(session.get_current_role()))\nprint('Database                    : {}'.format(session.get_current_database()))\nprint('Schema                      : {}'.format(session.get_current_schema()))\nprint('Warehouse                   : {}'.format(session.get_current_warehouse()))\nprint('Snowflake version           : {}'.format(snowflake_environment[0][1]))\nprint('Snowpark for Python version : {}.{}.{}'.format(VERSION[0],VERSION[1],VERSION[2]))\nprint('Snowflake ML version        : {}.{}.{}'.format(version.VERSION[0],version.VERSION[2],version.VERSION[4]))",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "32a4dd5a-5bb6-423a-ae43-356d1f0701db",
   "metadata": {
    "language": "python",
    "name": "CELL27",
    "collapsed": false
   },
   "outputs": [],
   "source": "#import warnings\n#warnings.filterwarnings(\"ignore\", message=\".*The 'nopython' keyword.*\")",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "0950f225-de4e-45c8-bc9c-b9539521deeb",
   "metadata": {
    "name": "cell18",
    "collapsed": false
   },
   "source": "## Save metrics in the model registry"
  },
  {
   "cell_type": "code",
   "id": "69c3b8b1-2f4e-4d70-be53-f73940ac9a4e",
   "metadata": {
    "language": "python",
    "name": "cell24",
    "collapsed": false
   },
   "outputs": [],
   "source": "from snowflake.ml.registry import Registry\n\n#get data\ntest = session.read.table(\"RAJIV.DOCAI.FRAUD_TRAINING_DATA\")\n\n#get model\nreg = Registry(session=session, database_name=\"FRAUD_FEATURE_STORE\", schema_name=\"FEATURE_STORE\")\nm = reg.get_model(\"FRAUD_MODEL\").version(\"V1\")",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "4ccc3c4b-91e9-4fa4-996b-e93b1242b4e0",
   "metadata": {
    "language": "python",
    "name": "cell34",
    "collapsed": false
   },
   "outputs": [],
   "source": "test.show()",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "c0a44cf3-8b91-47f0-b659-8b215fc2889d",
   "metadata": {
    "name": "cell25",
    "collapsed": false
   },
   "source": "Save relevant metrics to the model in the registry"
  },
  {
   "cell_type": "code",
   "id": "130f030c-2d31-4d8c-9e22-b4ac583dfefd",
   "metadata": {
    "language": "python",
    "name": "cell26",
    "collapsed": false
   },
   "outputs": [],
   "source": "from snowflake.ml.modeling.metrics import accuracy_score\npreds = m.run(test, function_name='predict')\n\nacc = accuracy_score(df=preds, y_true_col_names='TRANSACTION_FLAG', y_pred_col_names='\"OUTPUT_TRANSACTION_FLAG\"')\nprint(\"Accuracy: \",acc)\n\nm.set_metric(\"Accuracy\", value=acc)",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "704f6ba9-fc60-4aac-836b-9a51996f1f70",
   "metadata": {
    "language": "python",
    "name": "cell27",
    "collapsed": false
   },
   "outputs": [],
   "source": "reg.show_models()",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "3beb3c30-4a1c-42f3-b80b-1e807584df91",
   "metadata": {
    "name": "cell15",
    "collapsed": false
   },
   "source": "## Let's evaluate Model Performance over time using our Feature Store"
  },
  {
   "cell_type": "code",
   "id": "6bbfdec3-c7c9-4093-82ff-1ddde4ee298f",
   "metadata": {
    "language": "python",
    "name": "cell7",
    "collapsed": false
   },
   "outputs": [],
   "source": "spine_df = session.table('FRAUD_FEATURE_STORE.FEATURE_STORE.FR_TRANSACTIONS')\nspine_df.to_pandas().tail()",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "a042d4b8-fd25-4169-aea3-15d7392032f4",
   "metadata": {
    "language": "python",
    "name": "cell16",
    "collapsed": false
   },
   "outputs": [],
   "source": "from snowflake.ml.feature_store import FeatureStore\nfs = FeatureStore(\n    session=session, \n    database=\"FRAUD_FEATURE_STORE\",\n    name=\"FEATURE_STORE\",\n    default_warehouse=session.get_current_warehouse(),\n)\n\nalert_fv = FeatureView = fs.get_feature_view(\n    name='ALERT_FEATURES',\n    version='V2'\n)\n\naccount_fv = FeatureView = fs.get_feature_view(\n    name='ACCOUNT_FEATURES',\n    version='V2'\n)",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "aad93c82-2cca-480c-ba59-9343d9506d53",
   "metadata": {
    "language": "python",
    "name": "cell33",
    "collapsed": false
   },
   "outputs": [],
   "source": "training_data = fs.generate_dataset(\n    name='FRAUD_CLASSIFICATION',\n    version='V17',\n    spine_df=spine_df,\n    features=[alert_fv, account_fv],\n    spine_timestamp_col=\"TRANSACTION_TIME_UTC\",\n    spine_label_cols = [\"TRANSACTION_FLAG\"],\n)",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "1c4c6949-2514-4d9f-a72d-cc134bca3a20",
   "metadata": {
    "language": "python",
    "name": "cell32",
    "collapsed": false
   },
   "outputs": [],
   "source": "## Magic to make it all interesting\ntraining_data_pd = training_data.read.to_pandas()\ntraining_data_pd['WEEK_OF_YEAR'] = training_data_pd['TRANSACTION_TIME_UTC'].dt.isocalendar().week\nweek_21_transactions = training_data_pd[training_data_pd['WEEK_OF_YEAR'] == 21]\nfraudulent_samples = week_21_transactions.sample(n=1000, random_state=42,replace=True) \nfraudulent_samples['TRANSACTION_AMOUNT'] *= np.random.uniform(2, 5, size=len(fraudulent_samples))  # Increase amount by 10% to 50%\ntraining_data_pd = pd.concat([training_data_pd, fraudulent_samples], ignore_index=True)\ntraining_data_pd['IS_FRAUD'] = ((training_data_pd['TRANSACTION_AMOUNT'] > 2000) | (training_data_pd['TRANSACTION_FLAG'] > 30)).astype(int)\ntraining_data_pd['IS_FRAUD'] = 1 - training_data_pd['IS_FRAUD']\n\nfiltered_df = training_data_pd[training_data_pd['WEEK_OF_YEAR'].isin([17, 21])]\n",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "64cd0b1a-5489-43bd-88ab-f604ca2d3dc3",
   "metadata": {
    "name": "cell10",
    "collapsed": false
   },
   "source": "Pulled a dataset from the feature store on the latest transactions along with account information, alert information, and liklihood of fraud"
  },
  {
   "cell_type": "code",
   "id": "9c42b401-79dd-49b0-88f9-5a6176921403",
   "metadata": {
    "language": "python",
    "name": "cell6",
    "collapsed": false
   },
   "outputs": [],
   "source": "filtered_df.tail()",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "9f42df5d-cccf-4175-bb76-80530b23b4e0",
   "metadata": {
    "name": "cell9",
    "collapsed": false
   },
   "source": "### Let's explore the drift in the likelihood of Fraud"
  },
  {
   "cell_type": "code",
   "id": "bdad15c0-6e7b-49a7-bdc6-4051f36e0fcf",
   "metadata": {
    "language": "python",
    "name": "cell30",
    "collapsed": false
   },
   "outputs": [],
   "source": "fraud_week17 = filtered_df[filtered_df['WEEK_OF_YEAR'] == 17]['IS_FRAUD']\nfraud_week21 = filtered_df[filtered_df['WEEK_OF_YEAR'] == 21]['IS_FRAUD']\nplt.figure(figsize=(10, 6))\nsns.kdeplot(data=fraud_week17, label='Week 17', fill=True, color='blue')\nsns.kdeplot(data=fraud_week21, label='Week 21', fill=True, color='red')\nplt.title('Kernel Density Estimate of Fraud for Weeks 17 and 21')\nplt.xlabel('Likelihood of Fraud')\nplt.ylabel('Density')\nplt.legend()\nplt.show()",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "2daed1f5-93e7-4a08-bec9-4756df95de39",
   "metadata": {
    "language": "python",
    "name": "cell35",
    "collapsed": false
   },
   "outputs": [],
   "source": "def calculate_psi(expected_array, actual_array, buckets=10):\n    breakpoints = np.linspace(0, 100, buckets + 1)\n    breakpoints = np.percentile(expected_array, breakpoints)\n\n    expected_counts = np.histogram(expected_array, breakpoints)[0]\n    actual_counts = np.histogram(actual_array, breakpoints)[0]\n\n    epsilon = 1e-10\n    expected_percents = (expected_counts+epsilon) / expected_counts.sum()\n    actual_percents = (actual_counts+epsilon) / actual_counts.sum()\n\n    psi_values = (actual_percents - expected_percents) * np.log(actual_percents / expected_percents)\n    psi_values = np.where(np.isnan(psi_values), 0, psi_values)  # Replace NaNs with 0\n\n    total_psi = np.sum(psi_values)\n    return total_psi",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "59b9e3ba-aa22-4f73-9404-38e535cc0942",
   "metadata": {
    "language": "python",
    "name": "cell31",
    "collapsed": false
   },
   "outputs": [],
   "source": "psi = calculate_psi(fraud_week17,fraud_week21)\nprint(f\"Fraud PSI: {psi}\")",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "6b816f54-64f2-4b7b-91a3-51a533ff9b88",
   "metadata": {
    "name": "cell2",
    "collapsed": false
   },
   "source": "## Lets investigate some of the features"
  },
  {
   "cell_type": "code",
   "id": "04e9292b-60e9-4030-8ab6-0a3b23551edf",
   "metadata": {
    "language": "python",
    "name": "cell37",
    "collapsed": false
   },
   "outputs": [],
   "source": "trans_amount17 = filtered_df[filtered_df['WEEK_OF_YEAR'] == 17]['TRANSACTION_AMOUNT']\ntrans_amount21 = filtered_df[filtered_df['WEEK_OF_YEAR'] == 21]['TRANSACTION_AMOUNT']\nplt.figure(figsize=(10, 6))\nsns.kdeplot(data=trans_amount17, label='Week 17', fill=True, color='blue')\nsns.kdeplot(data=trans_amount21, label='Week 21', fill=True, color='red')\nplt.title('Kernel Density Estimate of TRANSACTION_AMOUNT for Weeks 17 and 21')\nplt.xlabel('TRANSACTION_AMOUNT')\nplt.ylabel('Density')\nplt.legend()\nplt.show()\n\npsi = calculate_psi(filtered_df[filtered_df['WEEK_OF_YEAR'] == 17]['TRANSACTION_AMOUNT'], filtered_df[filtered_df['WEEK_OF_YEAR'] == 21]['TRANSACTION_AMOUNT'])\nprint(f\"PSI: {psi}\")",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "6bfc722b-bbc3-46d0-a230-16528806e831",
   "metadata": {
    "language": "python",
    "name": "cell1",
    "collapsed": false
   },
   "outputs": [],
   "source": "plt.figure(figsize=(10, 6))\nalert_week17 = filtered_df[filtered_df['WEEK_OF_YEAR'] == 17]['AVG60MIN_ALERT_MM_H']\nalert_week21 = filtered_df[filtered_df['WEEK_OF_YEAR'] == 21]['AVG60MIN_ALERT_MM_H']\nsns.kdeplot(data=alert_week17, label='Week 17', fill=True, color='blue')\nsns.kdeplot(data=alert_week21, label='Week 21', fill=True, color='red')\nplt.title('Kernel Density Estimate of 60 Minute Average for Alerts for Weeks 17 and 21')\nplt.xlabel('Averaging alerts over 60 minute window for a merchant')\nplt.ylabel('Density')\nplt.legend()\nplt.show()\n\npsi = calculate_psi(filtered_df[filtered_df['WEEK_OF_YEAR'] == 17]['AVG60MIN_ALERT_MM_H'], filtered_df[filtered_df['WEEK_OF_YEAR'] == 21]['AVG60MIN_ALERT_MM_H'])\nprint(f\"PSI: {psi}\")",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "dbbdd926-36a6-4ffc-a8d4-74375669a1f8",
   "metadata": {
    "name": "cell5",
    "collapsed": false
   },
   "source": "### Let's try an open source package - evidently"
  },
  {
   "cell_type": "code",
   "id": "1d7d1e7b-6bfa-4795-bdbb-9fc7db287adf",
   "metadata": {
    "language": "python",
    "name": "cell40",
    "collapsed": false
   },
   "outputs": [],
   "source": "from evidently import ColumnMapping\n\nfrom evidently.report import Report\nfrom evidently.metrics.base_metric import generate_column_metrics\nfrom evidently.metric_preset import DataDriftPreset, TargetDriftPreset\nfrom evidently.metrics import *\n\nfrom evidently.test_suite import TestSuite\nfrom evidently.tests.base_test import generate_column_tests\nfrom evidently.test_preset import DataStabilityTestPreset, NoTargetPerformanceTestPreset\nfrom evidently.tests import *",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "008e68e1-c105-47d1-8abb-afbf27185b96",
   "metadata": {
    "language": "python",
    "name": "cell42",
    "collapsed": false
   },
   "outputs": [],
   "source": "week17 = filtered_df[filtered_df['WEEK_OF_YEAR'] == 17][['TRANSACTION_AMOUNT','TRANSACTION_FLAG','AVG60MIN_ALERT_MM_H']]\nweek21 = filtered_df[filtered_df['WEEK_OF_YEAR'] == 21][['TRANSACTION_AMOUNT','TRANSACTION_FLAG','AVG60MIN_ALERT_MM_H']]",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "4cc1e83d-8e5f-41d9-986f-6033dff2f797",
   "metadata": {
    "language": "python",
    "name": "cell39",
    "collapsed": false
   },
   "outputs": [],
   "source": "report = Report(metrics=[\n    DataDriftPreset(), \n])\n\nreport.run(reference_data=week17, current_data=week21)",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "f71e5598-15c2-432d-bc54-6b5b40c7aa98",
   "metadata": {
    "language": "python",
    "name": "cell41",
    "collapsed": false
   },
   "outputs": [],
   "source": "report.as_dict()",
   "execution_count": null
  }
 ]
}