3_1_K-mean_clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e0c13b8a-dc85-46cc-80e3-302aa493517e",
   "metadata": {},
   "source": [
    "**Lesson exercise notebook for Spark-for-Machine-Learning-AI Course**\n",
    "* Created by Kevin Chao (kevinchao@gmail.com)\n",
    "* Latest updated on Feb 14, 2024"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddc7beb6-61cb-4a23-aaa6-35087bc04119",
   "metadata": {},
   "source": [
    "* K-means:\n",
    "    * useful for data exploration of small and mid-sized datasets\n",
    "* Hierarchical clustering with Bisecting K-means:\n",
    "  * useful for clustering large datasets\n",
    "* Different algorithms may find different cluster centers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c4041d46-0f09-4c6a-8f76-c8b7ba3325c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init()\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName(\"YourAppName\").getOrCreate()\n",
    "\n",
    "from pyspark.ml.linalg import Vectors\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.clustering import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "57c53669-00bf-4db9-b259-0962c80d4635",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note: If inferSchema is set to false or not specified, Spark will default to treating all columns as strings.\n",
    "cluster_df = spark.read.csv(\"Exercise_Files/Ch03/03_02/clustering_dataset.csv\", header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2abe6277-bbb5-4863-bee5-cd235d41df0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(col1=7, col2=4, col3=1)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster_df.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "63b18664-b1dd-487d-83b8-6eba977c2862",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----+----+\n",
      "|col1|col2|col3|\n",
      "+----+----+----+\n",
      "|   7|   4|   1|\n",
      "|   7|   7|   9|\n",
      "|   7|   9|   6|\n",
      "|   1|   6|   5|\n",
      "|   6|   7|   7|\n",
      "|   7|   9|   4|\n",
      "|   7|  10|   6|\n",
      "|   7|   8|   2|\n",
      "|   8|   3|   8|\n",
      "|   4|  10|   5|\n",
      "|   7|   4|   5|\n",
      "|   7|   8|   4|\n",
      "|   2|   5|   1|\n",
      "|   2|   6|   2|\n",
      "|   2|   3|   8|\n",
      "|   3|   9|   1|\n",
      "|   4|   2|   9|\n",
      "|   1|   7|   1|\n",
      "|   6|   2|   3|\n",
      "|   4|   1|   9|\n",
      "+----+----+----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cluster_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c2c5b7a3-a27b-4199-bacc-3ba7e0aff2da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----+----+\n",
      "|col1|col2|col3|\n",
      "+----+----+----+\n",
      "|   7|   4|   1|\n",
      "|   7|   7|   9|\n",
      "|   7|   9|   6|\n",
      "|   1|   6|   5|\n",
      "|   6|   7|   7|\n",
      "|   7|   9|   4|\n",
      "|   7|  10|   6|\n",
      "|   7|   8|   2|\n",
      "|   8|   3|   8|\n",
      "|   4|  10|   5|\n",
      "|   7|   4|   5|\n",
      "|   7|   8|   4|\n",
      "|   2|   5|   1|\n",
      "|   2|   6|   2|\n",
      "|   2|   3|   8|\n",
      "|   3|   9|   1|\n",
      "|   4|   2|   9|\n",
      "|   1|   7|   1|\n",
      "|   6|   2|   3|\n",
      "|   4|   1|   9|\n",
      "|   4|   8|   5|\n",
      "|   6|   6|   7|\n",
      "|   4|   6|   2|\n",
      "|   8|   1|   1|\n",
      "|   7|   5|  10|\n",
      "|  17|  25|  21|\n",
      "|  15|  23|  32|\n",
      "|  42|  25|  45|\n",
      "|  41|  47|  21|\n",
      "|  37|  20|  27|\n",
      "|  40|  18|  26|\n",
      "|  41|  28|  50|\n",
      "|  32|  25|  40|\n",
      "|  24|  29|  35|\n",
      "|  47|  18|  47|\n",
      "|  36|  42|  45|\n",
      "|  49|  29|  15|\n",
      "|  47|  39|  22|\n",
      "|  38|  27|  25|\n",
      "|  45|  23|  40|\n",
      "|  23|  36|  19|\n",
      "|  47|  40|  50|\n",
      "|  37|  30|  40|\n",
      "|  42|  48|  41|\n",
      "|  29|  31|  21|\n",
      "|  36|  39|  48|\n",
      "|  50|  24|  31|\n",
      "|  42|  44|  37|\n",
      "|  37|  39|  46|\n",
      "|  22|  40|  30|\n",
      "|  17|  29|  41|\n",
      "|  85| 100|  69|\n",
      "|  68|  76|  67|\n",
      "|  76|  70|  93|\n",
      "|  62|  66|  91|\n",
      "|  83|  93|  76|\n",
      "|  95|  72|  63|\n",
      "|  75|  94|  95|\n",
      "|  83|  72|  80|\n",
      "|  93|  87|  76|\n",
      "|  86|  93|  63|\n",
      "|  97|  82|  75|\n",
      "|  61|  74|  74|\n",
      "|  84|  90| 100|\n",
      "|  77|  67|  97|\n",
      "|  61|  82|  73|\n",
      "|  81|  60|  69|\n",
      "|  67|  80|  98|\n",
      "|  94|  82|  60|\n",
      "|  69|  73|  74|\n",
      "|  74|  96|  80|\n",
      "|  86|  62|  61|\n",
      "|  88|  68|  95|\n",
      "|  99|  67|  80|\n",
      "|  76|  95|  70|\n",
      "+----+----+----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Show all data, the data are withij three groups\n",
    "cluster_df.show(75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9bf147bd-998a-40ca-8ce6-3b95042f1cee",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorAssembler = VectorAssembler(inputCols=[\"col1\", \"col2\", \"col3\"], outputCol=\"features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ec252669-0b33-47a9-85d8-01fc48d9eedd",
   "metadata": {},
   "outputs": [],
   "source": [
    "vcluster_df = vectorAssembler.transform(cluster_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5dbf36ec-80d4-4059-9a56-f84a1d20030f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----+----+--------------+\n",
      "|col1|col2|col3|      features|\n",
      "+----+----+----+--------------+\n",
      "|   7|   4|   1| [7.0,4.0,1.0]|\n",
      "|   7|   7|   9| [7.0,7.0,9.0]|\n",
      "|   7|   9|   6| [7.0,9.0,6.0]|\n",
      "|   1|   6|   5| [1.0,6.0,5.0]|\n",
      "|   6|   7|   7| [6.0,7.0,7.0]|\n",
      "|   7|   9|   4| [7.0,9.0,4.0]|\n",
      "|   7|  10|   6|[7.0,10.0,6.0]|\n",
      "|   7|   8|   2| [7.0,8.0,2.0]|\n",
      "|   8|   3|   8| [8.0,3.0,8.0]|\n",
      "|   4|  10|   5|[4.0,10.0,5.0]|\n",
      "|   7|   4|   5| [7.0,4.0,5.0]|\n",
      "|   7|   8|   4| [7.0,8.0,4.0]|\n",
      "|   2|   5|   1| [2.0,5.0,1.0]|\n",
      "|   2|   6|   2| [2.0,6.0,2.0]|\n",
      "|   2|   3|   8| [2.0,3.0,8.0]|\n",
      "|   3|   9|   1| [3.0,9.0,1.0]|\n",
      "|   4|   2|   9| [4.0,2.0,9.0]|\n",
      "|   1|   7|   1| [1.0,7.0,1.0]|\n",
      "|   6|   2|   3| [6.0,2.0,3.0]|\n",
      "|   4|   1|   9| [4.0,1.0,9.0]|\n",
      "+----+----+----+--------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vcluster_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "0b222a7a-1fff-4fbe-a449-57c3a683d834",
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeans = KMeans().setK(3)  # Set the numbers of cluster (k) to 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "029cf471-e9cd-40bc-916f-2ff2492bc31f",
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeans = kmeans.setSeed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4d6e98e6-e69a-47e4-af4e-219cc2b547c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "kmodel = kmeans.fit(vcluster_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "fec6be5f-2834-45e6-97d5-eb30ff18e429",
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = kmodel.clusterCenters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "19369912-9890-44a7-9f3c-078d7a03ab47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([35.88461538, 31.46153846, 34.42307692]),\n",
       " array([80.        , 79.20833333, 78.29166667]),\n",
       " array([5.12, 5.84, 4.84])]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show the center of each cluster\n",
    "centers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ab5ab2c-086f-4789-a662-27ba2322de6b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2e1c83ae-704d-4ef6-a992-8dbabc224952",
   "metadata": {},
   "source": [
    "* Hierarchical clustering with Bisecting K-means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e44d5461-19a2-411a-b9da-b21a3546b594",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----+----+--------------+\n",
      "|col1|col2|col3|      features|\n",
      "+----+----+----+--------------+\n",
      "|   7|   4|   1| [7.0,4.0,1.0]|\n",
      "|   7|   7|   9| [7.0,7.0,9.0]|\n",
      "|   7|   9|   6| [7.0,9.0,6.0]|\n",
      "|   1|   6|   5| [1.0,6.0,5.0]|\n",
      "|   6|   7|   7| [6.0,7.0,7.0]|\n",
      "|   7|   9|   4| [7.0,9.0,4.0]|\n",
      "|   7|  10|   6|[7.0,10.0,6.0]|\n",
      "|   7|   8|   2| [7.0,8.0,2.0]|\n",
      "|   8|   3|   8| [8.0,3.0,8.0]|\n",
      "|   4|  10|   5|[4.0,10.0,5.0]|\n",
      "|   7|   4|   5| [7.0,4.0,5.0]|\n",
      "|   7|   8|   4| [7.0,8.0,4.0]|\n",
      "|   2|   5|   1| [2.0,5.0,1.0]|\n",
      "|   2|   6|   2| [2.0,6.0,2.0]|\n",
      "|   2|   3|   8| [2.0,3.0,8.0]|\n",
      "|   3|   9|   1| [3.0,9.0,1.0]|\n",
      "|   4|   2|   9| [4.0,2.0,9.0]|\n",
      "|   1|   7|   1| [1.0,7.0,1.0]|\n",
      "|   6|   2|   3| [6.0,2.0,3.0]|\n",
      "|   4|   1|   9| [4.0,1.0,9.0]|\n",
      "+----+----+----+--------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vcluster_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "42bc7abb-0a6e-43ba-8ffb-f4b8a23f676d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.clustering import BisectingKMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0420eff5-4a5a-4ac6-a890-85f5537c4321",
   "metadata": {},
   "outputs": [],
   "source": [
    "bkmeans = BisectingKMeans().setK(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "39b7d26f-cdcf-4378-9c41-1d12f98308a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "bkmeans = bkmeans.setSeed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ad575b6f-f897-47e8-8cde-ce7638858986",
   "metadata": {},
   "outputs": [],
   "source": [
    "bkmodel = bkmeans.fit(vcluster_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a9aadce3-9994-49bf-bffa-aa9c6cd7ce35",
   "metadata": {},
   "outputs": [],
   "source": [
    "bkcenters = bkmodel.clusterCenters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a568a9bd-f9a8-4207-8ef2-e339f90af2b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([5.12, 5.84, 4.84]),\n",
       " array([35.88461538, 31.46153846, 34.42307692]),\n",
       " array([80.        , 79.20833333, 78.29166667])]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bkcenters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "106e1e7f-160c-4f34-bba6-bf7c1633cc6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([35.88461538, 31.46153846, 34.42307692]),\n",
       " array([80.        , 79.20833333, 78.29166667]),\n",
       " array([5.12, 5.84, 4.84])]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "centers  # The centers may not be the same for a larger dataset, but will be very similar"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}