2_1_normlize_numeric_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3942a668-5b94-4bfc-810b-aa7c5db51e94",
   "metadata": {},
   "source": [
    "**Lesson exercise notebook for Spark-for-Machine-Learning-AI Course**\n",
    "* Created by Kevin Chao (kevinchao@gmail.com)\n",
    "* Latest updated on Feb 14, 2024"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f471019-1ab3-496f-9221-2f6a0f34e813",
   "metadata": {},
   "source": [
    "* Numeric: Standardize\n",
    "    * Map data values from their original range to -1 to 1\n",
    "    * Mean value of 0\n",
    "    * Normally distributed with standard deviation of 1\n",
    "    * Used when attributes have different scales and ML algorithms assueme a normal distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dbd8dd79-c126-457c-86cd-1510db6b4ed3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init()\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName(\"YourAppName\").getOrCreate()\n",
    "\n",
    "from pyspark.ml.feature import MinMaxScaler\n",
    "from pyspark.ml.linalg import Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8f080416-a16e-45bf-9202-707b5be8b04f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dataframe with \"id\" and \"features\"\n",
    "features_df = spark.createDataFrame([\n",
    "    (1, Vectors.dense([10.0, 10000.0, 1.0]),),\n",
    "    (2, Vectors.dense([20.0, 30000.0, 2.0]),),\n",
    "    (3, Vectors.dense([30.0, 40000.0, 3.0]),) ], \n",
    "    [\"id\", \"features\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7fdc333d-fa77-4144-bb84-23ec1bbd5ef5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(id=1, features=DenseVector([10.0, 10000.0, 1.0]))]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_df.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d9883124-fd24-4c2a-a65a-722d2c90fb56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a scaler object\n",
    "feature_scaler = MinMaxScaler(inputCol=\"features\", outputCol=\"sfeatures\", min=0.0, max=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d90e3ee9-0d3f-4f09-9246-300b5c1c6f0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the input features and create a model\n",
    "smodel = feature_scaler.fit(features_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "eb04b3a8-d3fc-475f-8396-24749c3043db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply the transformation to tyhe dataframe\n",
    "sfeatures_df = smodel.transform(features_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "db67adeb-8320-4221-ada3-7d19c913c4c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(id=1, features=DenseVector([10.0, 10000.0, 1.0]), sfeatures=SparseVector(3, {})),\n",
       " Row(id=2, features=DenseVector([20.0, 30000.0, 2.0]), sfeatures=DenseVector([0.5, 0.6667, 0.5])),\n",
       " Row(id=3, features=DenseVector([30.0, 40000.0, 3.0]), sfeatures=DenseVector([1.0, 1.0, 1.0]))]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sfeatures_df.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7bfc09e0-8a60-40dd-b0fd-aad9f1971523",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+--------------------+\n",
      "|          features|           sfeatures|\n",
      "+------------------+--------------------+\n",
      "|[10.0,10000.0,1.0]|           (3,[],[])|\n",
      "|[20.0,30000.0,2.0]|[0.5,0.6666666666...|\n",
      "|[30.0,40000.0,3.0]|       [1.0,1.0,1.0]|\n",
      "+------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sfeatures_df.select(\"features\", \"sfeatures\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bbd3ad78-34a4-4813-86ab-4c8e79689fb6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 36:=================================================>        (6 + 1) / 7]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+--------------------+\n",
      "|          features|           sfeatures|\n",
      "+------------------+--------------------+\n",
      "|[10.0,10000.0,1.0]|     [0.0, 0.0, 0.0]|\n",
      "|[20.0,30000.0,2.0]|[0.5, 0.666666666...|\n",
      "|[30.0,40000.0,3.0]|     [1.0, 1.0, 1.0]|\n",
      "+------------------+--------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "from pyspark.sql.functions import udf\n",
    "from pyspark.sql.types import ArrayType, DoubleType\n",
    "\n",
    "# Define a UDF to convert SparseVector to DenseVector\n",
    "sparse_to_dense_udf = udf(lambda sv: sv.toArray().tolist(), ArrayType(DoubleType()))\n",
    "\n",
    "# Apply the UDF to the sfeatures column\n",
    "sfeatures_df2 = sfeatures_df.withColumn(\"sfeatures\", sparse_to_dense_udf(\"sfeatures\"))\n",
    "\n",
    "sfeatures_df2.select(\"features\", \"sfeatures\").show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}