hobit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "13932029",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FILES IN THIS DIRECTORY\n",
      "['.bashrc', '.bash_logout', '.profile', 'Untitled.ipynb', 'confluentinc-kafka-connect-mqtt-1.7.2.zip', '.ipython', '.cache', '.npm', '.bash_history', '.local', '.ipynb_checkpoints', 'config.json', '.ivy2', '.jupyter', 'jars', '.conda', '.config', '.wget-hsts', 'work']\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession, Row\n",
    "from pyspark.sql.functions import from_json\n",
    "from pyspark.sql.types import *\n",
    "from pyspark.conf import SparkConf\n",
    "from pyspark.context import SparkContext\n",
    "import time as timer\n",
    "import argparse\n",
    "import datetime\n",
    "import json\n",
    "from pyspark.sql.functions import col, abs, mean, expr, substring, udf\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.regression import LinearRegression\n",
    "from pyspark.ml.regression import RandomForestRegressor\n",
    "\n",
    "from pyspark.sql.functions import udf\n",
    "from pyspark.sql.types import DoubleType\n",
    "\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "#import lightgbm as lgb\n",
    "\n",
    "schema = StructType(\n",
    "    [\n",
    "        StructField(\"num_date_time\", StringType()),\n",
    "        StructField(\"건물번호\", StringType()),\n",
    "        StructField(\"일시\", StringType()),\n",
    "        StructField(\"기온(C)\", StringType()),\n",
    "        StructField(\"강수량(mm)\", StringType()),\n",
    "        StructField(\"풍속(m/s)\", StringType()),\n",
    "        StructField(\"습도(%)\", StringType()),\n",
    "        StructField(\"일조(hr)\", StringType()),\n",
    "        StructField(\"일사(MJ/m2)\", StringType()),\n",
    "        StructField(\"전력소비량(kWh)\", StringType()),\n",
    "    ]\n",
    ")\n",
    "\n",
    "schema2 = StructType(\n",
    "    [\n",
    "        StructField(\"num_date_time\", StringType()),\n",
    "        StructField(\"건물번호\", StringType()),\n",
    "        StructField(\"일시\", StringType()),\n",
    "        StructField(\"기온(C)\", StringType()),\n",
    "        StructField(\"강수량(mm)\", StringType()),\n",
    "        StructField(\"풍속(m/s)\", StringType()),\n",
    "        StructField(\"습도(%)\", StringType()),\n",
    "    ]\n",
    ")\n",
    "\n",
    "schema3 = StructType(\n",
    "    [\n",
    "        StructField(\"건물번호\", StringType()),\n",
    "        StructField(\"건물유형\", StringType()),\n",
    "        StructField(\"연면적(m2)\", StringType()),\n",
    "        StructField(\"냉방면적(m2)\", StringType()),\n",
    "        StructField(\"태양광용량(kW)\", StringType()),\n",
    "        StructField(\"ESS저장용량(kWh)\", StringType()),\n",
    "        StructField(\"PCS용량(kW)\", StringType()),\n",
    "    ]\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "print(\"FILES IN THIS DIRECTORY\")\n",
    "print(os.listdir(os.getcwd()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "QT7_IBn9QwYz",
   "metadata": {
    "id": "QT7_IBn9QwYz"
   },
   "source": [
    "## Fixed Random-Seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "KgyHxME7QwYz",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-11T08:41:57.009050Z",
     "start_time": "2022-07-11T08:41:57.006949Z"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1683473189486,
     "user": {
      "displayName": "이승윤",
      "userId": "17995305622900705411"
     },
     "user_tz": -540
    },
    "id": "KgyHxME7QwYz"
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "seed_everything(42) # Seed 고정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c8aa0773",
   "metadata": {},
   "outputs": [],
   "source": [
    "# config.json 파일 읽기\n",
    "with open(\"config.json\", \"r\") as f:\n",
    "    config = json.load(f)\n",
    "\n",
    "jar_urls = \",\".join(config[\"KAFKA_JAR_URLS\"])\n",
    "repartition_num = config[\"NUM_EXECUTORS\"] * config[\"EXECUTOR_CORES\"] * 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "513a9c19",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: An illegal reflective access operation has occurred\n",
      "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/opt/conda/lib/python3.9/site-packages/pyspark/jars/spark-unsafe_2.12-3.2.4.jar) to constructor java.nio.DirectByteBuffer(long,int)\n",
      "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n",
      "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
      "WARNING: All illegal access operations will be denied in a future release\n",
      "24/05/21 08:56:18 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "24/05/21 08:56:18 WARN SparkConf: Total executor cores: 24 is not divisible by cores per executor: 16, the left cores: 8 will not be allocated\n"
     ]
    }
   ],
   "source": [
    "# SparkSession 생성\n",
    "spark = (\n",
    "    SparkSession.builder.master(\"spark://spark-master-service:7077\")\n",
    "    .config(\"spark.driver.bindAddress\", \"0.0.0.0\")\n",
    "    .config(\"spark.driver.host\", \"10.42.2.146\")\n",
    "    .config(\"spark.driver.port\", \"39337\")\n",
    "    .config(\"spark.cores.max\", \"24\")\n",
    "    .config(\"spark.executor.instances\", config[\"NUM_EXECUTORS\"])\n",
    "    .config(\"spark.executor.cores\", config[\"EXECUTOR_CORES\"])\n",
    "    .config(\"spark.executor.memory\", config[\"EXECUTOR_MEMORY\"])\n",
    "    .config(\"spark.defaul.parallelism\", repartition_num)\n",
    "    .config(\"spark.sql.shuffle.partitions\", repartition_num)\n",
    "    .config(\"spark.jars\", jar_urls)  # JAR 파일 포함\n",
    "    .appName(\"asdf\")\n",
    "    .getOrCreate()\n",
    ")\n",
    "\n",
    "sc = spark.sparkContext\n",
    "sc.setLogLevel(\"ERROR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5a544621",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://10.42.2.146:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.2.4</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>spark://spark-master-service:7077</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>asdf</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=spark://spark-master-service:7077 appName=asdf>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d4144ecc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Spark configuration:\n",
      "spark.app.id = app-20240521085619-0391\n",
      "spark.app.initial.jar.urls = spark://10.42.2.146:39337/jars/spark-streaming-kafka-0-10_2.12-3.2.4.jar,spark://10.42.2.146:39337/jars/hadoop-client-runtime-3.3.1.jar,spark://10.42.2.146:39337/jars/jsr305-3.0.0.jar,spark://10.42.2.146:39337/jars/htrace-core4-4.1.0-incubating.jar,spark://10.42.2.146:39337/jars/commons-logging-1.1.3.jar,spark://10.42.2.146:39337/jars/kafka-clients-2.8.1.jar,spark://10.42.2.146:39337/jars/spark-token-provider-kafka-0-10_2.12-3.2.4.jar,spark://10.42.2.146:39337/jars/commons-pool2-2.6.2.jar,spark://10.42.2.146:39337/jars/hadoop-client-api-3.3.1.jar,spark://10.42.2.146:39337/jars/spark-sql-kafka-0-10_2.12-3.2.4.jar\n",
      "spark.app.name = asdf\n",
      "spark.app.startTime = 1716281778650\n",
      "spark.cores.max = 24\n",
      "spark.defaul.parallelism = 96\n",
      "spark.driver.bindAddress = 0.0.0.0\n",
      "spark.driver.host = 10.42.2.146\n",
      "spark.driver.port = 39337\n",
      "spark.executor.cores = 16\n",
      "spark.executor.id = driver\n",
      "spark.executor.instances = 3\n",
      "spark.executor.memory = 24G\n",
      "spark.jars = jars/jsr305-3.0.0.jar,jars/commons-logging-1.1.3.jar,jars/commons-pool2-2.6.2.jar,jars/hadoop-client-api-3.3.1.jar,jars/hadoop-client-runtime-3.3.1.jar,jars/htrace-core4-4.1.0-incubating.jar,jars/kafka-clients-2.8.1.jar,jars/spark-token-provider-kafka-0-10_2.12-3.2.4.jar,jars/spark-sql-kafka-0-10_2.12-3.2.4.jar,jars/spark-streaming-kafka-0-10_2.12-3.2.4.jar\n",
      "spark.master = spark://spark-master-service:7077\n",
      "spark.rdd.compress = True\n",
      "spark.repl.local.jars = file:///home/jovyan/jars/jsr305-3.0.0.jar,file:///home/jovyan/jars/commons-logging-1.1.3.jar,file:///home/jovyan/jars/commons-pool2-2.6.2.jar,file:///home/jovyan/jars/hadoop-client-api-3.3.1.jar,file:///home/jovyan/jars/hadoop-client-runtime-3.3.1.jar,file:///home/jovyan/jars/htrace-core4-4.1.0-incubating.jar,file:///home/jovyan/jars/kafka-clients-2.8.1.jar,file:///home/jovyan/jars/spark-token-provider-kafka-0-10_2.12-3.2.4.jar,file:///home/jovyan/jars/spark-sql-kafka-0-10_2.12-3.2.4.jar,file:///home/jovyan/jars/spark-streaming-kafka-0-10_2.12-3.2.4.jar\n",
      "spark.serializer.objectStreamReset = 100\n",
      "spark.sql.shuffle.partitions = 96\n",
      "spark.sql.warehouse.dir = file:/home/jovyan/spark-warehouse\n",
      "spark.submit.deployMode = client\n",
      "spark.submit.pyFiles = \n",
      "spark.ui.showConsoleProgress = true\n"
     ]
    }
   ],
   "source": [
    "print(\"Current Spark configuration:\")\n",
    "for key, value in sorted(sc._conf.getAll(), key=lambda x: x[0]):\n",
    "    print(f\"{key} = {value}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a1afdca2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- createTime: string (nullable = true)\n",
      " |-- 건물번호: string (nullable = true)\n",
      " |-- 건물유형: string (nullable = true)\n",
      " |-- 연면적(m2): string (nullable = true)\n",
      " |-- 냉방면적(m2): string (nullable = true)\n",
      " |-- 태양광용량(kW): string (nullable = true)\n",
      " |-- ESS저장용량(kWh): string (nullable = true)\n",
      " |-- PCS용량(kW): string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 그냥 가져오기\n",
    "building_sdf = (\n",
    "    spark.read.format(\"kafka\")\n",
    "    .option(\"kafka.bootstrap.servers\", \"my-cluster-kafka-bootstrap.kafka.svc:9092\")\n",
    "    .option(\"subscribe\", \"building-jy\")\n",
    "    .option(\"kafka.group.id\", \"my_consumer_group\")\n",
    "    .load()\n",
    ")  # 밀리초 단위 에포치 시간endingTimestamp\n",
    "building_sdf = building_sdf.selectExpr(\"CAST(value AS STRING)\", \"CAST(timestamp AS STRING)\")\n",
    "building_sdf = building_sdf.withColumnRenamed(\"timestamp\", \"createTime\")\n",
    "building_sdf = building_sdf.withColumn(\"value\", from_json(building_sdf[\"value\"], schema3))\n",
    "\n",
    "\n",
    "for field in schema3.fields:\n",
    "    building_sdf = building_sdf.withColumn(field.name, building_sdf[\"value.\" + field.name])\n",
    "building_sdf = building_sdf.drop(\"value\")\n",
    "# 이거쓰면 df가 repartition_num 수만큼 쪼개져서 병렬처리가능한 상태가 됨.\n",
    "building_sdf = building_sdf.repartition(repartition_num)\n",
    "\n",
    "\n",
    "building_sdf.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "962ca250",
   "metadata": {},
   "outputs": [],
   "source": [
    "building_sdf = building_sdf.select(\n",
    "    building_sdf[\"createTime\"],\n",
    "    building_sdf[\"건물번호\"],\n",
    "    building_sdf[\"건물유형\"],\n",
    "    building_sdf[\"연면적(m2)\"],\n",
    "    building_sdf[\"냉방면적(m2)\"],\n",
    "    building_sdf[\"태양광용량(kW)\"],\n",
    "    building_sdf[\"ESS저장용량(kWh)\"],\n",
    "    building_sdf[\"PCS용량(kW)\"]\n",
    ") \\\n",
    ".withColumn(\"createTime\", building_sdf[\"createTime\"].cast(StringType())) \\\n",
    ".withColumn(\"건물번호\", building_sdf[\"건물번호\"].cast(IntegerType())) \\\n",
    ".withColumn(\"건물유형\", building_sdf[\"건물유형\"].cast(StringType())) \\\n",
    ".withColumn(\"연면적(m2)\", building_sdf[\"연면적(m2)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"냉방면적(m2)\", building_sdf[\"냉방면적(m2)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"태양광용량(kW)\", building_sdf[\"태양광용량(kW)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"ESS저장용량(kWh)\", building_sdf[\"ESS저장용량(kWh)\"].cast(IntegerType())) \\\n",
    ".withColumn(\"PCS용량(kW)\", building_sdf[\"PCS용량(kW)\"].cast(IntegerType()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "37006873",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 0:>                                                          (0 + 3) / 3]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------+--------------+----------+------------+--------------+----------------+-----------+\n",
      "|          createTime|건물번호|      건물유형|연면적(m2)|냉방면적(m2)|태양광용량(kW)|ESS저장용량(kWh)|PCS용량(kW)|\n",
      "+--------------------+--------+--------------+----------+------------+--------------+----------------+-----------+\n",
      "|2024-05-13 11:15:...|      65|        아파트|  183839.0|         0.0|          null|            null|       null|\n",
      "|2024-05-13 11:15:...|      14|      건물기타|  16844.16|    14102.92|          56.0|            null|       null|\n",
      "|2024-05-16 13:12:...|      36|    데이터센터|   8816.49|     8816.49|          null|            null|       null|\n",
      "|2024-05-13 11:15:...|      46|          병원|  85869.49|     78675.0|        100.56|            null|       null|\n",
      "|2024-05-13 11:14:...|      30|        대학교|  155785.0|    106305.0|          20.0|            null|       null|\n",
      "|2024-05-16 13:12:...|      40|백화점및아울렛|   58483.0|     40775.0|          null|            null|       null|\n",
      "|2024-05-21 01:23:...|      89|      할인마트|   78454.0|     13362.0|          null|            null|       null|\n",
      "|2024-05-13 11:15:...|      70|        연구소|  141014.0|    120000.0|         28.44|            null|       null|\n",
      "|2024-05-13 11:15:...|      99|  호텔및리조트|   53499.0|     40636.0|          null|            null|       null|\n",
      "|2024-05-13 11:15:...|      62|        아파트|  233187.0|    178666.0|          null|            null|       null|\n",
      "|2024-05-21 01:23:...|      51|          병원|  88242.76|     80566.0|          null|            null|       null|\n",
      "|2024-05-21 01:24:...|      33|    데이터센터|   28059.0|     20397.0|          null|            null|       null|\n",
      "|2024-05-16 13:12:...|      97|  호텔및리조트|  55144.67|     25880.0|          null|            null|       null|\n",
      "|2024-05-21 01:24:...|       3|      건물기타|  171243.0|    113950.0|          40.0|            null|       null|\n",
      "|2024-05-13 11:14:...|      82|  지식산업센터|  225651.0|    114772.0|          null|            null|       null|\n",
      "|2024-05-21 01:23:...|      10|      건물기타|   31062.0|     18637.0|          null|            null|       null|\n",
      "|2024-05-13 11:15:...|       3|      건물기타|  171243.0|    113950.0|          40.0|            null|       null|\n",
      "|2024-05-13 11:15:...|       7|      건물기타| 101711.52|     41341.1|          null|             800|        300|\n",
      "|2024-05-13 11:14:...|      51|          병원|  88242.76|     80566.0|          null|            null|       null|\n",
      "|2024-05-13 11:15:...|      11|      건물기타|  45956.56|    13190.98|         447.2|            null|       null|\n",
      "+--------------------+--------+--------------+----------+------------+--------------+----------------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "building_sdf.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0d37249c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- createTime: string (nullable = true)\n",
      " |-- num_date_time: string (nullable = true)\n",
      " |-- 건물번호: string (nullable = true)\n",
      " |-- 일시: string (nullable = true)\n",
      " |-- 기온(C): string (nullable = true)\n",
      " |-- 강수량(mm): string (nullable = true)\n",
      " |-- 풍속(m/s): string (nullable = true)\n",
      " |-- 습도(%): string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 그냥 가져오기\n",
    "test_sdf = (\n",
    "    spark.read.format(\"kafka\")\n",
    "    .option(\"kafka.bootstrap.servers\", \"my-cluster-kafka-bootstrap.kafka.svc:9092\")\n",
    "    .option(\"subscribe\", \"test2-jy\")\n",
    "    .option(\"kafka.group.id\", \"my_consumer_group\")\n",
    "    .load()\n",
    ")  # 밀리초 단위 에포치 시간endingTimestamp\n",
    "test_sdf = test_sdf.selectExpr(\"CAST(value AS STRING)\", \"CAST(timestamp AS STRING)\")\n",
    "test_sdf = test_sdf.withColumnRenamed(\"timestamp\", \"createTime\")\n",
    "test_sdf = test_sdf.withColumn(\"value\", from_json(test_sdf[\"value\"], schema2))\n",
    "\n",
    "\n",
    "for field in schema2.fields:\n",
    "    test_sdf = test_sdf.withColumn(field.name, test_sdf[\"value.\" + field.name])\n",
    "test_sdf = test_sdf.drop(\"value\")\n",
    "# 이거쓰면 df가 repartition_num 수만큼 쪼개져서 병렬처리가능한 상태가 됨.\n",
    "test_sdf = test_sdf.repartition(repartition_num)\n",
    "\n",
    "\n",
    "test_sdf.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "53523147",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sdf = test_sdf.select(\n",
    "    test_sdf[\"createTime\"],\n",
    "    test_sdf[\"num_date_time\"],\n",
    "    test_sdf[\"건물번호\"],\n",
    "    test_sdf[\"일시\"],\n",
    "    test_sdf[\"기온(C)\"],\n",
    "    test_sdf[\"강수량(mm)\"],\n",
    "    test_sdf[\"풍속(m/s)\"],\n",
    "    test_sdf[\"습도(%)\"]\n",
    ") \\\n",
    ".withColumn(\"createTime\", test_sdf[\"createTime\"].cast(StringType())) \\\n",
    ".withColumn(\"num_date_time\", test_sdf[\"num_date_time\"].cast(StringType())) \\\n",
    ".withColumn(\"건물번호\", test_sdf[\"건물번호\"].cast(IntegerType())) \\\n",
    ".withColumn(\"일시\", test_sdf[\"일시\"].cast(StringType())) \\\n",
    ".withColumn(\"기온(C)\", test_sdf[\"기온(C)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"강수량(mm)\", test_sdf[\"강수량(mm)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"풍속(m/s)\", test_sdf[\"풍속(m/s)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"습도(%)\", test_sdf[\"습도(%)\"].cast(IntegerType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5d6ce09c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 5:>                                                          (0 + 3) / 3]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------+--------+-----------+-------+----------+---------+-------+\n",
      "|          createTime| num_date_time|건물번호|       일시|기온(C)|강수량(mm)|풍속(m/s)|습도(%)|\n",
      "+--------------------+--------------+--------+-----------+-------+----------+---------+-------+\n",
      "|2024-05-21 01:23:...|74_20220827 08|      74|20220827 08|   18.3|       0.0|      2.1|     77|\n",
      "|2024-05-21 01:23:...|57_20220827 03|      57|20220827 03|   20.7|       2.0|      2.7|     91|\n",
      "|2024-05-21 01:23:...|24_20220827 09|      24|20220827 09|   19.5|       0.0|      2.4|     63|\n",
      "|2024-05-21 01:23:...|94_20220829 04|      94|20220829 04|   21.8|       0.0|      1.3|     66|\n",
      "|2024-05-21 01:23:...| 5_20220829 15|       5|20220829 15|   22.2|       0.0|      1.8|     73|\n",
      "|2024-05-21 01:23:...|35_20220831 04|      35|20220831 04|   19.8|       0.0|      1.2|     94|\n",
      "|2024-05-21 01:23:...|67_20220827 04|      67|20220827 04|   22.6|       0.0|      3.2|     79|\n",
      "|2024-05-21 01:23:...|63_20220830 20|      63|20220830 20|   20.0|       2.6|      1.0|     95|\n",
      "|2024-05-21 01:23:...|18_20220825 23|      18|20220825 23|   20.5|       0.0|      0.4|     87|\n",
      "|2024-05-21 01:23:...|92_20220830 04|      92|20220830 04|   18.0|       5.5|      1.0|     99|\n",
      "|2024-05-21 01:23:...|94_20220829 16|      94|20220829 16|   21.8|       0.0|      3.4|     75|\n",
      "|2024-05-21 01:23:...|82_20220831 02|      82|20220831 02|   19.1|       0.0|      1.8|     94|\n",
      "|2024-05-21 01:23:...|21_20220826 02|      21|20220826 02|   20.1|       0.0|      0.7|     79|\n",
      "|2024-05-21 01:23:...|86_20220829 20|      86|20220829 20|   20.6|       0.0|      1.9|     73|\n",
      "|2024-05-21 01:23:...|40_20220826 22|      40|20220826 22|   23.1|       0.0|      1.0|     88|\n",
      "|2024-05-21 01:23:...|33_20220825 03|      33|20220825 03|   22.1|       0.0|      1.3|     78|\n",
      "|2024-05-21 01:23:...|62_20220830 07|      62|20220830 07|   18.0|       1.2|      3.1|     98|\n",
      "|2024-05-21 01:23:...|19_20220825 03|      19|20220825 03|   22.1|       0.0|      1.3|     78|\n",
      "|2024-05-21 01:23:...|19_20220831 22|      19|20220831 22|   21.1|       0.0|      1.0|     83|\n",
      "|2024-05-21 01:23:...|26_20220826 15|      26|20220826 15|   29.1|       0.0|      2.4|     57|\n",
      "+--------------------+--------------+--------+-----------+-------+----------+---------+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "test_sdf.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e719173b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- createTime: string (nullable = true)\n",
      " |-- num_date_time: string (nullable = true)\n",
      " |-- 건물번호: string (nullable = true)\n",
      " |-- 일시: string (nullable = true)\n",
      " |-- 기온(C): string (nullable = true)\n",
      " |-- 강수량(mm): string (nullable = true)\n",
      " |-- 풍속(m/s): string (nullable = true)\n",
      " |-- 습도(%): string (nullable = true)\n",
      " |-- 일조(hr): string (nullable = true)\n",
      " |-- 일사(MJ/m2): string (nullable = true)\n",
      " |-- 전력소비량(kWh): string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 그냥 가져오기\n",
    "train_sdf = (\n",
    "    spark.read.format(\"kafka\")\n",
    "    .option(\"kafka.bootstrap.servers\", \"my-cluster-kafka-bootstrap.kafka.svc:9092\")\n",
    "    .option(\"subscribe\", \"test-jy\")\n",
    "    .option(\"kafka.group.id\", \"my_consumer_group\")\n",
    "    .load()\n",
    ")  # 밀리초 단위 에포치 시간endingTimestamp\n",
    "\n",
    "train_sdf = train_sdf.selectExpr(\"CAST(value AS STRING)\", \"CAST(timestamp AS STRING)\")\n",
    "train_sdf = train_sdf.withColumnRenamed(\"timestamp\", \"createTime\")\n",
    "train_sdf = train_sdf.withColumn(\"value\", from_json(train_sdf[\"value\"], schema))\n",
    "\n",
    "\n",
    "for field in schema.fields:\n",
    "    train_sdf = train_sdf.withColumn(field.name, train_sdf[\"value.\" + field.name])\n",
    "train_sdf = train_sdf.drop(\"value\")\n",
    "# 이거쓰면 df가 repartition_num 수만큼 쪼개져서 병렬처리가능한 상태가 됨.\n",
    "\n",
    "\n",
    "train_sdf = train_sdf.repartition(repartition_num)\n",
    "\n",
    "train_sdf.printSchema()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "761214c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sdf = train_sdf.select(\n",
    "    train_sdf[\"createTime\"],\n",
    "    train_sdf[\"num_date_time\"],\n",
    "    train_sdf[\"건물번호\"],\n",
    "    train_sdf[\"일시\"],\n",
    "    train_sdf[\"기온(C)\"],\n",
    "    train_sdf[\"강수량(mm)\"],\n",
    "    train_sdf[\"풍속(m/s)\"],\n",
    "    train_sdf[\"습도(%)\"],\n",
    "    train_sdf[\"일조(hr)\"],\n",
    "    train_sdf[\"일사(MJ/m2)\"],\n",
    "    train_sdf[\"전력소비량(kWh)\"],\n",
    ") \\\n",
    ".withColumn(\"createTime\", train_sdf[\"createTime\"].cast(StringType())) \\\n",
    ".withColumn(\"num_date_time\", train_sdf[\"num_date_time\"].cast(StringType())) \\\n",
    ".withColumn(\"건물번호\", train_sdf[\"건물번호\"].cast(IntegerType())) \\\n",
    ".withColumn(\"일시\", train_sdf[\"일시\"].cast(StringType())) \\\n",
    ".withColumn(\"기온(C)\", train_sdf[\"기온(C)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"강수량(mm)\", train_sdf[\"강수량(mm)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"풍속(m/s)\", train_sdf[\"풍속(m/s)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"습도(%)\", train_sdf[\"습도(%)\"].cast(IntegerType())) \\\n",
    ".withColumn(\"일조(hr)\", train_sdf[\"일조(hr)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"일사(MJ/m2)\", train_sdf[\"일사(MJ/m2)\"].cast(DoubleType())) \\\n",
    ".withColumn(\"전력소비량(kWh)\", train_sdf[\"전력소비량(kWh)\"].cast(DoubleType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c196be6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 8:===================>                                       (1 + 2) / 3]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------+--------+-----------+-------+----------+---------+-------+--------+-----------+---------------+\n",
      "|          createTime| num_date_time|건물번호|       일시|기온(C)|강수량(mm)|풍속(m/s)|습도(%)|일조(hr)|일사(MJ/m2)|전력소비량(kWh)|\n",
      "+--------------------+--------------+--------+-----------+-------+----------+---------+-------+--------+-----------+---------------+\n",
      "|2024-05-21 01:18:...|81_20220706 17|      81|20220706 17|   33.2|      null|      2.1|     62|     0.3|       1.49|        2403.36|\n",
      "|2024-05-21 01:18:...|69_20220615 05|      69|20220615 05|   15.2|       0.8|      2.2|     93|    null|       null|         3369.9|\n",
      "|2024-05-21 01:18:...|76_20220806 16|      76|20220806 16|   34.5|      null|      1.6|     64|     0.6|        1.6|        1127.88|\n",
      "|2024-05-21 01:17:...|26_20220620 05|      26|20220620 05|   20.6|      null|      0.3|     96|    null|       null|        2698.08|\n",
      "|2024-05-21 01:17:...|41_20220729 20|      41|20220729 20|   30.9|       0.0|      2.4|     69|     0.0|       0.04|         4482.6|\n",
      "|2024-05-21 01:18:...|78_20220617 23|      78|20220617 23|   22.0|      null|      0.3|     88|    null|       null|         965.28|\n",
      "|2024-05-21 01:17:...|11_20220823 22|      11|20220823 22|   24.5|      null|      1.0|     95|    null|       null|         1722.0|\n",
      "|2024-05-21 01:17:...|20_20220802 17|      20|20220802 17|   29.0|      null|      1.5|     76|     0.6|       null|        3267.84|\n",
      "|2024-05-21 01:17:...|22_20220805 19|      22|20220805 19|   28.2|      null|      1.7|     85|     0.4|       0.28|        1229.04|\n",
      "|2024-05-21 01:18:...|65_20220718 15|      65|20220718 15|   25.3|       0.1|      1.5|     99|     0.0|        0.4|         455.04|\n",
      "|2024-05-21 01:18:...|83_20220731 10|      83|20220731 10|   27.8|       1.1|      1.5|     87|     0.0|       0.62|          921.0|\n",
      "|2024-05-21 01:18:...|97_20220628 03|      97|20220628 03|   29.3|      null|      5.9|     71|    null|       null|         785.88|\n",
      "|2024-05-21 01:17:...|12_20220722 17|      12|20220722 17|   27.4|      null|      2.4|     66|     0.0|       0.81|         1713.6|\n",
      "|2024-05-21 01:17:...|21_20220608 18|      21|20220608 18|   23.5|      null|      1.0|     43|     0.2|       0.83|        1853.76|\n",
      "|2024-05-21 01:18:...|77_20220801 01|      77|20220801 01|   25.9|       0.5|      1.4|     98|    null|       null|        1034.64|\n",
      "|2024-05-21 01:17:...|20_20220807 18|      20|20220807 18|   30.1|      null|      0.8|     73|     0.3|       null|        1061.76|\n",
      "|2024-05-21 01:18:...|94_20220706 16|      94|20220706 16|   32.8|      null|      3.0|     63|     0.1|       1.57|        3590.16|\n",
      "|2024-05-21 01:16:...| 4_20220728 03|       4|20220728 03|   26.2|      null|      1.7|     71|    null|       null|         661.14|\n",
      "|2024-05-21 01:17:...|31_20220609 15|      31|20220609 15|   25.1|      null|      1.6|     45|     0.9|       2.21|         1773.0|\n",
      "|2024-05-21 01:17:...| 8_20220706 04|       8|20220706 04|   23.8|      null|      0.3|    100|    null|       null|        1481.04|\n",
      "+--------------------+--------------+--------+-----------+-------+----------+---------+-------+--------+-----------+---------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "train_sdf.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1430079a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_test_split(df, th):\n",
    "    df = df.na.drop(subset=['일시'])\n",
    "    train = df.filter(col('일시').substr(1, 8).cast(IntegerType()) < th)\n",
    "    test = df.filter(col('일시').substr(1, 8).cast(IntegerType()) >= th)\n",
    "    return train, test\n",
    "\n",
    "def preprocess_x(df):\n",
    "    to_remove_columns = ['num_date_time', '일시', '일조(hr)', '일사(MJ/m2)', '전력소비량(kWh)']\n",
    "    df = df.fillna(0)\n",
    "    \n",
    "    # 시계열 특성을 학습에 반영하기 위해 일시를 월, 일, 시간으로 나눕니다\n",
    "    df = df.withColumn('month', substring('일시', 5, 2).cast(IntegerType()))\n",
    "    df = df.withColumn('day', substring('일시', 7, 2).cast(IntegerType()))\n",
    "    df = df.withColumn('time', substring('일시', 10, 2).cast(IntegerType()))\n",
    "    \n",
    "    df = df.join(building_sdf.select('건물번호', '건물유형', '연면적(m2)'), on='건물번호', how='left')\n",
    "    \n",
    "    # '건물유형'을 카테고리형 코드로 변환\n",
    "    building_type_indexer = StringIndexer(inputCol='건물유형', outputCol='건물유형_index')\n",
    "    df = building_type_indexer.fit(df).transform(df)\n",
    "    \n",
    "    # 불필요한 컬럼 삭제\n",
    "    for c in to_remove_columns:\n",
    "        if c in df.columns:\n",
    "            df = df.drop(c)\n",
    "    return df\n",
    "\n",
    "\n",
    "date_th = 20220820"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d14b45cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train DataFrame shape: 192000 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation DataFrame shape: 12000 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train X shape: 192000 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Y shape: 192000 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation X shape: 12000 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 59:===================>                                      (1 + 2) / 3]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Y shape: 12000 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "\n",
    "train_df, valid_df = train_test_split(train_sdf, date_th)\n",
    "\n",
    "# 데이터 분할 후 각 데이터프레임의 크기를 확인합니다.\n",
    "print(\"Train DataFrame shape:\", train_df.count(), len(train_df.columns))\n",
    "print(\"Validation DataFrame shape:\", valid_df.count(), len(valid_df.columns))\n",
    "\n",
    "train_x = preprocess_x(train_df)\n",
    "train_y = train_df.select(\"전력소비량(kWh)\")\n",
    "\n",
    "# 전처리 후 데이터프레임의 크기를 확인합니다.\n",
    "print(\"Train X shape:\", train_df.count(), len(train_df.columns))\n",
    "print(\"Train Y shape:\", train_y.count(), len(train_y.columns))\n",
    "\n",
    "valid_x = preprocess_x(valid_df)\n",
    "valid_y = valid_df.select(\"전력소비량(kWh)\")\n",
    "\n",
    "print(\"Validation X shape:\", valid_df.count(), len(valid_df.columns))\n",
    "print(\"Validation Y shape:\", valid_y.count(), len(valid_y.columns))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fde9a701",
   "metadata": {},
   "outputs": [],
   "source": [
    "#spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4d5c633e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 65:===================>                                      (1 + 2) / 3]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+-------+----------+---------+-------+-----+---+----+------------+----------+--------------+\n",
      "|건물번호|          createTime|기온(C)|강수량(mm)|풍속(m/s)|습도(%)|month|day|time|    건물유형|연면적(m2)|건물유형_index|\n",
      "+--------+--------------------+-------+----------+---------+-------+-----+---+----+------------+----------+--------------+\n",
      "|      25|2024-05-21 01:17:...|   23.7|       0.0|      2.8|     96|    7| 14|   3|      대학교| 528014.05|           2.0|\n",
      "|      25|2024-05-21 01:17:...|   23.7|       0.0|      2.8|     96|    7| 14|   3|      대학교| 528014.05|           2.0|\n",
      "|      25|2024-05-21 01:17:...|   23.7|       0.0|      2.8|     96|    7| 14|   3|      대학교| 528014.05|           2.0|\n",
      "|      25|2024-05-21 01:17:...|   23.7|       0.0|      2.8|     96|    7| 14|   3|      대학교| 528014.05|           2.0|\n",
      "|      25|2024-05-21 01:17:...|   23.7|       0.0|      2.8|     96|    7| 14|   3|      대학교| 528014.05|           2.0|\n",
      "|      25|2024-05-21 01:17:...|   23.7|       0.0|      2.8|     96|    7| 14|   3|      대학교| 528014.05|           2.0|\n",
      "|      25|2024-05-21 01:17:...|   23.7|       0.0|      2.8|     96|    7| 14|   3|      대학교| 528014.05|           2.0|\n",
      "|      25|2024-05-21 01:17:...|   23.7|       0.0|      2.8|     96|    7| 14|   3|      대학교| 528014.05|           2.0|\n",
      "|      25|2024-05-21 01:17:...|   23.7|       0.0|      2.8|     96|    7| 14|   3|      대학교| 528014.05|           2.0|\n",
      "|       1|2024-05-21 01:16:...|   22.4|       3.8|      2.2|    100|    6| 30|   9|    건물기타|  110634.0|           0.0|\n",
      "|       1|2024-05-21 01:16:...|   22.4|       3.8|      2.2|    100|    6| 30|   9|    건물기타|  110634.0|           0.0|\n",
      "|       1|2024-05-21 01:16:...|   22.4|       3.8|      2.2|    100|    6| 30|   9|    건물기타|  110634.0|           0.0|\n",
      "|       1|2024-05-21 01:16:...|   22.4|       3.8|      2.2|    100|    6| 30|   9|    건물기타|  110634.0|           0.0|\n",
      "|       1|2024-05-21 01:16:...|   22.4|       3.8|      2.2|    100|    6| 30|   9|    건물기타|  110634.0|           0.0|\n",
      "|       1|2024-05-21 01:16:...|   22.4|       3.8|      2.2|    100|    6| 30|   9|    건물기타|  110634.0|           0.0|\n",
      "|       1|2024-05-21 01:16:...|   22.4|       3.8|      2.2|    100|    6| 30|   9|    건물기타|  110634.0|           0.0|\n",
      "|       1|2024-05-21 01:16:...|   22.4|       3.8|      2.2|    100|    6| 30|   9|    건물기타|  110634.0|           0.0|\n",
      "|       1|2024-05-21 01:16:...|   22.4|       3.8|      2.2|    100|    6| 30|   9|    건물기타|  110634.0|           0.0|\n",
      "|      94|2024-05-21 01:18:...|   27.3|       0.0|      2.5|     66|    6| 22|  11|호텔및리조트|   62156.0|          10.0|\n",
      "|      94|2024-05-21 01:18:...|   27.3|       0.0|      2.5|     66|    6| 22|  11|호텔및리조트|   62156.0|          10.0|\n",
      "+--------+--------------------+-------+----------+---------+-------+-----+---+----+------------+----------+--------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "train_x.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f570fdc6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 71:===================>                                      (1 + 2) / 3]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "|전력소비량(kWh)|\n",
      "+---------------+\n",
      "|        5351.52|\n",
      "|         1983.0|\n",
      "|          917.4|\n",
      "|        1009.08|\n",
      "|        1276.02|\n",
      "|        2023.68|\n",
      "|         1117.8|\n",
      "|          572.4|\n",
      "|         2352.0|\n",
      "|        3827.52|\n",
      "|         2325.6|\n",
      "|        1707.12|\n",
      "|        3120.12|\n",
      "|        2103.84|\n",
      "|        9758.16|\n",
      "|         4368.0|\n",
      "|        3750.72|\n",
      "|         416.64|\n",
      "|         240.66|\n",
      "|        1092.96|\n",
      "+---------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "train_y.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3da97b5a",
   "metadata": {},
   "source": [
    "### 0. Functions For validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "166f3a2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def SMAPE(y, pred):\n",
    "    smape = abs((y - pred))/((abs(y) + abs(pred)) / 2) * 100\n",
    "    smape = np.mean(smape)\n",
    "    return smape\n",
    "\n",
    "def mae(y, pred):\n",
    "    return np.mean(abs(y-pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6d7a7655",
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(valid_x, valid_y, model):\n",
    "    pred = model.predict(valid_x)\n",
    "    smape_score, mae_score = SMAPE(valid_y, pred), mae(valid_y, pred)\n",
    "    return smape_score, mae_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb086467",
   "metadata": {},
   "source": [
    "- n_estimators값이 300정도에서 모델 성능이 괜찮은 것을 확인할 수 있습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e9c573a",
   "metadata": {},
   "source": [
    "## 2. Multi Models by building num\n",
    "- 건물번호(1~100)별 모델을 각각 만들어 성능을 측정하였습니다.\n",
    "- 건물별로 building_info값은 같기 때문에 해당 데이터는 제외했습니다(건물번호, 건물유형, 연면적, 냉방면적)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "425a0f59",
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_multi(valid_x, valid_y, models):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        models: dict, {1: model1, 2: model2, ..., 100: model100}\n",
    "    \"\"\"\n",
    "    preds = []\n",
    "    for i in range(1, 101):\n",
    "        _x = valid_x.filter(col('건물번호') == i) \n",
    "        _x = _x.drop('건물번호', '건물유형', '연면적(m2)', '냉방면적(m2)')\n",
    "        \n",
    "        pred_df = models[i].transform(_x).select('predection')\n",
    "        pred_list = [row.predeiction for row in pred_df.collect()]\n",
    "        \n",
    "        preds.extend(pred_list)\n",
    "    \n",
    "    preds = np.array(preds)\n",
    "        \n",
    "    \n",
    "    smape_score = SMAPE(valid_y, preds_df)\n",
    "    mae_score = MAE(valid_y, preds_df)\n",
    "    \n",
    "    return smape_score, mae_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "98fce78d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_multiple_models(train_x, train_y, n_estimators=100):\n",
    "    models = {}\n",
    "    \n",
    "    \n",
    "    train_y = train_y.withColumnRenamed('전력소비량(kWh)', 'label')\n",
    "    \n",
    "    for i in tqdm(range(1, 101)):\n",
    "        _x = train_x.filter(col('건물번호') == i)\n",
    "        _y = train_y.filter(col('건물번호') == i).select('label')\n",
    "        \n",
    "        _x = _x.drop('건물번호', '건물유형', '연면적(m2)', '냉방면적(m2)', 'createTime')\n",
    "        \n",
    "        feature_cols = [c for c in _x.columns if c != 'label']\n",
    "        assembler = VectorAssembler(inputCols = feature_cols, outputCol = \"features\")\n",
    "        _x = assembler.transform(_x).select(\"features\")\n",
    "        \n",
    "        train_data = _x.join(_y)\n",
    "        \n",
    "        rf = RandomForestRegressor(featuresCol='features', labelCol='label', numTrees=n_estimators)\n",
    "        model = rf.fit(train_data)\n",
    "        \n",
    "        models[i] = model\n",
    "        \n",
    "        \n",
    "    return models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6548d905",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 234:=========================>                            (46 + 16) / 96]\r"
     ]
    }
   ],
   "source": [
    "models1 = train_multiple_models(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed559390",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models1)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c050020",
   "metadata": {},
   "source": [
    "#### 2-1 hyper parameter 조절\n",
    "- 건물별 모델을 따로 만드는게 성능이 약간 더 좋았고(15.798 -> 13.13), multiple models의 경우 n_estimators를 작게 하는게 오히려 성능이 좋았습니다.\n",
    "- 아마 모델별 데이터의 양이 작아져서 학습횟수를 크게하는게 과적합을 야기하는 것으로 생각됩니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c554075",
   "metadata": {},
   "outputs": [],
   "source": [
    "models2 = train_multiple_models(train_x, train_y, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe0898af",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models2)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42cb23f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "models3 = train_multiple_models(train_x, train_y, 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5ff9625",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models3)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25cdcabc",
   "metadata": {},
   "source": [
    "## 3. Add Features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cbff80c",
   "metadata": {},
   "source": [
    "### A. Weekday\n",
    "- [가벼운 DATA EDA입니다.](https://dacon.io/competitions/official/236125/codeshare/8500?page=1&dtype=recent)에서 주말과 주중 전력사용량이 유의미한 차이를 보임을 확인할 수 있었습니다. 그래서 날짜를 이용해 요일데이터를 추출해 feature로 사용하였습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "302b3a73",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "844cb9e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_datetime(s):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        s: ex) '20220601 01'\n",
    "    Returns:\n",
    "        weekday: 0~6(int), 0: 월요일, 1: 화요일, ...\n",
    "    \"\"\"\n",
    "    s = s.split()[0]  # 20220601\n",
    "    date = datetime.datetime.strptime(s, '%Y%m%d')\n",
    "    weekday = date.weekday()  # \n",
    "    return weekday"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4015b053",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_origin_ = train_origin.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce90c20b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_origin_['Weekday'] = train_origin_.apply(lambda x:to_datetime(x['일시']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ff06f5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_origin_.sample(5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c222b91",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, valid데이터 재생성\n",
    "train_df, valid_df = train_test_split(train_origin_, 20220820)\n",
    "\n",
    "train_x = preprocess_x(train_df)\n",
    "train_y = train_df['전력소비량(kWh)']\n",
    "\n",
    "valid_x = preprocess_x(valid_df)\n",
    "valid_y = valid_df['전력소비량(kWh)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdda8d8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_f1 = train_multiple_models(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7d803fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_f1)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b3e92e6",
   "metadata": {},
   "source": [
    "#### A-1 hyper paramter 조절\n",
    "- weekday 데이터를 추가하니 validation SMAPE score가 13.13 -> 5.97 까지 하락하였습니다.\n",
    "- 참고로 해당 모델(validation score=5.976...)로 만든 submission을 dacon에 제출하였을때가 **best score였으며, score는 6.768이었습니다.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9675ea9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_f2 = train_multiple_models(train_x, train_y, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "990dd06c",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_f2)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68651299",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_f3 = train_multiple_models(train_x, train_y, 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfca79c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_f3)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4510455",
   "metadata": {},
   "source": [
    "### B. 전날과의 기온, 풍속 차이\n",
    "- 전 날과의 기온과 풍속, 습도의 차이값을 feature에 추가해보았습니다.\n",
    "- weekday는 그대로 사용하였습니다(train_origin_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b9fa85f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_origin_b = train_origin_.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "322d8adc",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1, 101):\n",
    "    df = train_origin_b[train_origin_b['건물번호'] == i]\n",
    "    train_origin_b.loc[df.index, '기온_gap'] = df['기온(C)'] - df.shift(1)['기온(C)']\n",
    "    train_origin_b.loc[df.index, '풍속_gap'] = df['풍속(m/s)'] - df.shift(1)['풍속(m/s)']\n",
    "    train_origin_b.loc[df.index, '습도_gap'] = df['습도(%)'] - df.shift(1)['습도(%)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b53195aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_origin_b.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d920749d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_origin_b['기온_gap'] = train_origin_b['기온_gap'].fillna(0)\n",
    "train_origin_b['풍속_gap'] = train_origin_b['풍속_gap'].fillna(0)\n",
    "train_origin_b['습도_gap'] = train_origin_b['습도_gap'].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c28031d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df, valid_df = train_test_split(train_origin_b, 20220820)\n",
    "\n",
    "train_x = preprocess_x(train_df)\n",
    "train_y = train_df['전력소비량(kWh)']\n",
    "\n",
    "valid_x = preprocess_x(valid_df)\n",
    "valid_y = valid_df['전력소비량(kWh)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2fc996f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8603a121",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_b1 = train_multiple_models(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fca42865",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_b1)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d3fce35",
   "metadata": {},
   "source": [
    "#### B-1 hyper paramter 조절\n",
    "- 그냥 weekday와 추가한 모델과 결과에서 큰 차이는 없었습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed2ae29b",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_b2 = train_multiple_models(train_x, train_y, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2acce9e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_b2)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7a633c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_b3 = train_multiple_models(train_x, train_y, 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a151d991",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_b3)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6634095",
   "metadata": {},
   "source": [
    "### C. 1주일 전 전력소모값\n",
    "- test 데이터의 가장 마지막 날짜(8월31일)과 train 데이터의 가장 마지막 날짜(8월24일)의 gap이 7일이므로, test 데이터의 경우 과거 7일전 이상의 전력소모값은 접근할 수 있습니다.\n",
    "- B에서 생성한 feature는 사용하지않고, weekday(train_origin_)만 사용하였습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53deb93f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_origin_c = train_origin_.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6733b027",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 7*24 # 7*24시간 전 전력소모량\n",
    "for i in range(1, 101):\n",
    "    df = train_origin_c[train_origin_c['건물번호'] == i]\n",
    "    train_origin_c.loc[df.index, f'{n}시간 전 전력소비량'] = df.shift(n)['전력소비량(kWh)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebe2a7a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df, valid_df = train_test_split(train_origin_c, 20220820)\n",
    "\n",
    "train_x = preprocess_x(train_df)\n",
    "train_y = train_df['전력소비량(kWh)']\n",
    "\n",
    "valid_x = preprocess_x(valid_df)\n",
    "valid_y = valid_df['전력소비량(kWh)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2d0f14f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = train_x[train_x['168시간 전 전력소비량'] != 0]  # NaN이 0으로 replace되어 0이 아닌 row들 삭제\n",
    "train_y = train_y[train_x.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e49738bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82464db6",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_c1 = train_multiple_models(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99c16d91",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_c1)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b61b127",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_c2 = train_multiple_models(train_x, train_y, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87c0a92b",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_c2)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a30102c",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_c3 = train_multiple_models(train_x, train_y, 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4695ee47",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_c3)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab47c072",
   "metadata": {},
   "source": [
    "- 오히려 score가 떨어졌습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3978c31a",
   "metadata": {},
   "source": [
    "### D. 7일전 24시간동안의 전력소모량\n",
    "- 8일전\\~7일전(191시간전\\~168시간전) 데이터를 feature로 추가해 사용해보았습니다.\n",
    "- 위와 마찬가지로 weekday만 있는 모델에서 feature를 추가하였습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a19de4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_origin_d = train_origin_.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38e38123",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1, 101):\n",
    "    df = train_origin_d[train_origin_d['건물번호'] == i]\n",
    "    fr, to = 7*24, 7*24+24\n",
    "    for n in range(fr, to):\n",
    "        train_origin_d.loc[df.index, f'{n}시간 전 전력소비량'] = df.shift(n)['전력소비량(kWh)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccef545e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df, valid_df = train_test_split(train_origin_d, 20220820)\n",
    "\n",
    "train_x = preprocess_x(train_df)\n",
    "train_y = train_df['전력소비량(kWh)']\n",
    "\n",
    "valid_x = preprocess_x(valid_df)\n",
    "valid_y = valid_df['전력소비량(kWh)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e876a0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b46e0a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = train_x[train_x['191시간 전 전력소비량'] != 0]  # NaN이 0으로 replace되어 0이 아닌 row들 삭제\n",
    "train_y = train_y[train_x.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f475555",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_d1 = train_multiple_models(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c42322d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_d1)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8cc3ab5",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_d2 = train_multiple_models(train_x, train_y, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a9c3562",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_d2)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82e3445a",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_d3 = train_multiple_models(train_x, train_y, 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8dc192c",
   "metadata": {},
   "outputs": [],
   "source": [
    "smape_score, mae_score = validate_multi(valid_x, valid_y, models_d3)\n",
    "print(f'SMAPE: {smape_score}\\nMAE: {mae_score}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31f94dde",
   "metadata": {},
   "source": [
    "- feature를 추가하는게 오히려 성능을 떨어뜨림을 확인할 수 있습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07fdc541",
   "metadata": {},
   "source": [
    "## Make Submission\n",
    "- 전체 학습 데이터셋을 이용해 모델 생성\n",
    "- 3-A(multiple models, add weekday)모델을 사용하였습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45ec9277",
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.read_csv('./data/sample_submission.csv')\n",
    "submission\n",
    "\n",
    "train_origin['Weekday'] = train_origin.apply(lambda x:to_datetime(x['일시']), axis=1)\n",
    "test_origin['Weekday'] = test_origin.apply(lambda x:to_datetime(x['일시']), axis=1)\n",
    "train_x_full = preprocess_x(train_origin)\n",
    "train_y_full = train_origin['전력소비량(kWh)']\n",
    "\n",
    "models = train_multiple_models(train_x_full, train_y_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b905b39b",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x_real = preprocess_x(test_origin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb69c753",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_real = []\n",
    "for i in tqdm(range(1, 101)):\n",
    "    _x = test_x_real[test_x_real['건물번호'] == i]\n",
    "    _x = _x.drop(columns=['건물번호', '건물유형', '연면적(m2)', '냉방면적(m2)'])\n",
    "    pred = models[i].predict(_x).tolist()\n",
    "    preds_real.extend(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f01a12db",
   "metadata": {},
   "outputs": [],
   "source": [
    "submission['answer'] = preds_real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a644a2ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60a5ccad",
   "metadata": {},
   "source": [
    "## 결론\n",
    "- 건물별 100개의 모델을 따로 생성하고, weekday를 추가한 모델이 가장 좋은 성능을 보였습니다.\n",
    "- validation score는 5.976, 실제 submission score는 6.76. \n",
    "- 두 값의 차이가 나는 이유는 best valiation score를 찾는 과정에서 약간의 overfitting이 발생하기도 했고, submssion에 사용하는 test 데이터의 설명변수(X)값들은 실제 값이 아닌 예측치라는 점도 영향을 미친것 같습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b89ed0f3",
   "metadata": {},
   "source": [
    "## TODO\n",
    "- 추가적인 feature를 고려해보았는데, 마땅히 떠오르는 아이디어가 없어서 모델의 hyper parameter 튜닝을 중점적으로 진행할 것 같습니다.\n",
    "- 그리고 valid_y와 preds를 비교해 성능이 잘 안나오는 모델의 건물번호, 시간대 등을 분석해 모델을 보완해나가는게 필요할 것 같습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "891ec694",
   "metadata": {},
   "source": [
    "## Epilog\n",
    "- 3-A 모델에서 예측을 잘 못한 모델을 한번 뽑아봤습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dadc9cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, valid데이터 재생성\n",
    "train_df, valid_df = train_test_split(train_origin, 20220820)\n",
    "train_x = preprocess_x(train_df)\n",
    "train_y = train_df['전력소비량(kWh)']\n",
    "valid_x = preprocess_x(valid_df)\n",
    "valid_y = valid_df['전력소비량(kWh)']\n",
    "models_f2 = train_multiple_models(train_x, train_y, 50)\n",
    "preds = []\n",
    "for i in range(1, 101):\n",
    "    _x = valid_x[valid_x['건물번호'] == i]\n",
    "    _x = _x.drop(columns=['건물번호', '건물유형', '연면적(m2)', '냉방면적(m2)'])\n",
    "    pred = models[i].predict(_x).tolist()\n",
    "    preds.extend(pred)\n",
    "preds = np.array(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32234b69",
   "metadata": {},
   "outputs": [],
   "source": [
    "eda = valid_x.copy()\n",
    "eda['pred'], eda['y'] = preds, valid_y\n",
    "eda['gap'] = (eda['y'] - eda['pred']) / ((eda['y'] + eda['pred'])/2)  # SMAPE가 target이기 때문에 scale 고려"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46856134",
   "metadata": {},
   "outputs": [],
   "source": [
    "eda.sort_values('gap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf53962b",
   "metadata": {},
   "outputs": [],
   "source": [
    "bads = eda.groupby('건물번호').agg(lambda x:np.mean(abs(x))).sort_values('gap').tail(5)\n",
    "goods = eda.groupby('건물번호').agg(lambda x:np.mean(abs(x))).sort_values('gap').head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5cae0d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "bads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82284160",
   "metadata": {},
   "outputs": [],
   "source": [
    "goods"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c046a2fc",
   "metadata": {},
   "source": [
    "- 95, 14번 건물 모델이 성능이 잘 안나왔고, 33, 32번 건물 모델 성능이 잘나왔네요.\n",
    "- 95, 14번처럼 매 시각마다 전력사용량이 크게 바뀌는 건물의 경우 좀더 복잡한 모델을 사용하는게 점수가 잘나올 듯 합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d359378",
   "metadata": {},
   "outputs": [],
   "source": [
    "eda[eda['건물번호']==95][['y', 'pred']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "023ee702",
   "metadata": {},
   "outputs": [],
   "source": [
    "eda[eda['건물번호']==14][['y', 'pred']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4a05646",
   "metadata": {},
   "outputs": [],
   "source": [
    "eda[eda['건물번호']==33][['y', 'pred']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da4ae141",
   "metadata": {},
   "outputs": [],
   "source": [
    "eda[eda['건물번호']==32][['y', 'pred']].plot()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "09436c3ae2a44a8ca51ee1103b963ada": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "29d20f64d430441ea9364b13dfef101b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4a9ef407319c43bcb20106c47f0db180": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_09436c3ae2a44a8ca51ee1103b963ada",
      "placeholder": "​",
      "style": "IPY_MODEL_f6d802990a2344db99bf282c663efa5a",
      "value": "100%"
     }
    },
    "4c356887768a4330ab0b0eec28984484": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "520b45d5f972463da632a32f96c71ee4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_29d20f64d430441ea9364b13dfef101b",
      "placeholder": "​",
      "style": "IPY_MODEL_e7d3b9b9d1224868988e665b4c677357",
      "value": " 1881/1881 [00:24&lt;00:00, 97.61it/s]"
     }
    },
    "53ccb4062ee94e0c84618143ff39ce05": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "55e857853048445b87cea2de151ba035": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "790f3379b0364c88ae6bdec739b97248": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7a0c9eaf1d1e4a56a195dff77a7ed5f9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bb35a24296ed455a872731cef7333ffb",
      "max": 5001,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_dc7da92e2f454f79b07c3ca86c3cc9d4",
      "value": 5001
     }
    },
    "911d4b1f888540eb96b8b6a48134f078": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "918b0c55f55f4b799fb41adb002c5f1f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_9a823940919d4e068c01e31276740b0f",
       "IPY_MODEL_9fe2d03a06474596b0547d0410da8064",
       "IPY_MODEL_520b45d5f972463da632a32f96c71ee4"
      ],
      "layout": "IPY_MODEL_911d4b1f888540eb96b8b6a48134f078"
     }
    },
    "9a823940919d4e068c01e31276740b0f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_53ccb4062ee94e0c84618143ff39ce05",
      "placeholder": "​",
      "style": "IPY_MODEL_df5b9e6414d449f0961eecec22b16991",
      "value": "100%"
     }
    },
    "9fe2d03a06474596b0547d0410da8064": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_790f3379b0364c88ae6bdec739b97248",
      "max": 1881,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e5a69cc464fe42f6be81a34ed57b1686",
      "value": 1881
     }
    },
    "a194d82dd952480387bf1f599a8e52e5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bb35a24296ed455a872731cef7333ffb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d5a7c1d85e63402ebd0c36cb6883ab33": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a194d82dd952480387bf1f599a8e52e5",
      "placeholder": "​",
      "style": "IPY_MODEL_4c356887768a4330ab0b0eec28984484",
      "value": " 5001/5001 [01:11&lt;00:00, 106.18it/s]"
     }
    },
    "dc7da92e2f454f79b07c3ca86c3cc9d4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "df5b9e6414d449f0961eecec22b16991": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e5a69cc464fe42f6be81a34ed57b1686": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "e7d3b9b9d1224868988e665b4c677357": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f6d802990a2344db99bf282c663efa5a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fca0079f6a714ac7885e1474711d7913": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_4a9ef407319c43bcb20106c47f0db180",
       "IPY_MODEL_7a0c9eaf1d1e4a56a195dff77a7ed5f9",
       "IPY_MODEL_d5a7c1d85e63402ebd0c36cb6883ab33"
      ],
      "layout": "IPY_MODEL_55e857853048445b87cea2de151ba035"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}