04 Extract Data-Climate Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "953e06a4-67f4-4b28-8895-0ccd364ca1c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xarray as xr\n",
    "import warnings\n",
    "from itertools import groupby\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ed029bc6-7440-4d0f-b09f-a919f07a080d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_workshop=pd.read_csv('/data/raw/Workshop_PUSO.csv',sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d8d6f0c9-fff2-48f8-b6a7-585b5bf480e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tahun  bulan\n",
       "22     1         47\n",
       "       2         32\n",
       "       3         13\n",
       "       4         21\n",
       "       5         14\n",
       "       6         27\n",
       "       7         13\n",
       "       8         25\n",
       "       9         19\n",
       "       10         3\n",
       "       11         5\n",
       "       12        12\n",
       "23     1          6\n",
       "       2          6\n",
       "       3          8\n",
       "       4          3\n",
       "       5         14\n",
       "       6         16\n",
       "       7         68\n",
       "       8         85\n",
       "       9         82\n",
       "       10       109\n",
       "       11        43\n",
       "       12        13\n",
       "Name: idsegmen, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_workshop.query('is_puso==1').groupby(['tahun','bulan'])['idsegmen'].agg('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "60a9a887-91ad-4a3e-ad52-06d5b520d99d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_workshop=data_workshop.sort_values(['tahun','bulan'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "312e3265-2463-45dd-b3d6-2adcc135cdc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "for g in ['mean','max','min']:\n",
    "    for h in ['total_precipitation_0_daily','2m_temperature_0_daily','2m_dewpoint_temperature_0_daily',\n",
    "          'convective_rain_rate_0_daily','potential_evaporation_0_daily','evaporation_0_daily','soil_temperature_level_1_0_daily',\n",
    "         'volumetric_soil_water_layer_1_0_daily']:\n",
    "        nc_list=[]\n",
    "        for i in [20,21,22,23]:\n",
    "            for j in range(1,13):\n",
    "                temp=data_workshop.query('tahun==@i and bulan==@j')\n",
    "                periode='20'+str(i)+'_'+str(j).zfill(2)\n",
    "                if g=='mean':\n",
    "                    netcdf_file = f\"/data/ksa/00_Code/12_Workshop_Puso/climate_data/{periode}_data/{h}-{g}.nc\"\n",
    "                else:\n",
    "                    netcdf_file = f\"/data/ksa/00_Code/12_Workshop_Puso/climate_data/{periode}_data_{g}/{h}-{g}.nc\"\n",
    "                dt=xr.open_dataset(netcdf_file)\n",
    "                nc_list.append(dt)\n",
    "        combined = xr.concat(nc_list, dim=\"valid_time\")\n",
    "        combined.to_netcdf(f\"/data/ksa/00_Code/12_Workshop_Puso/concated_climate_data/concated-ERA5-{h}-{g}.nc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "18ac7b08-76ef-436f-872a-175e28744cfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import calendar\n",
    "from datetime import datetime, timedelta\n",
    "def get_last_day_of_month(year, month):\n",
    "    _, last_day = calendar.monthrange(year, month)\n",
    "    return last_day\n",
    "def days_before(date_str, days):\n",
    "    # Parse the input date (format: YYYY-MM-DD)\n",
    "    date = datetime.strptime(date_str, \"%Y-%m-%d\")\n",
    "    # Subtract the number of days\n",
    "    new_date = date - timedelta(days=days)\n",
    "    return new_date.strftime(\"%Y-%m-%d\")  # Format the result as a string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a04fe6f8-b232-46df-b58b-e9d37281715f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_x=data_workshop.query('tahun==22').query('bulan==1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "55a2002a-b945-4e7b-bb50-89594f72be21",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_nc_value={\n",
    "    'total_precipitation_0_daily':'tp',\n",
    "    '2m_temperature_0_daily':'t2m',\n",
    "    '2m_dewpoint_temperature_0_daily':'d2m',\n",
    "    'convective_rain_rate_0_daily':'crr',\n",
    "    'potential_evaporation_0_daily':'pev',\n",
    "    'evaporation_0_daily':'e',\n",
    "    'soil_temperature_level_1_0_daily':'stl1',\n",
    "    'volumetric_soil_water_layer_1_0_daily':'swvl1'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "faa951f5-40a8-4ede-aea5-901729566165",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RUNNING_DATA=================\n",
      "***************************\n",
      "Tahun: 22\n",
      "Bulan: 1\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 2\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 3\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 4\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 5\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 6\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 7\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 8\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 9\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 10\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 11\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 12\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Tahun: 23\n",
      "Bulan: 1\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 2\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 3\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 4\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 5\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 6\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 7\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 8\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 9\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 10\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 11\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n",
      "Bulan: 12\n",
      "Data: total_precipitation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: 2m_dewpoint_temperature_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: convective_rain_rate_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: potential_evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: evaporation_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: soil_temperature_level_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "Data: volumetric_soil_water_layer_1_0_daily\n",
      "Daily Statistics:\n",
      "mean|max|min|\n",
      "\n",
      "***************************\n"
     ]
    }
   ],
   "source": [
    "data_compile=pd.DataFrame()\n",
    "print('RUNNING_DATA=================')\n",
    "print('***************************')\n",
    "for i in data_workshop.tahun.unique():\n",
    "    print('Tahun:',i)\n",
    "    for j in data_workshop.bulan.unique():\n",
    "        print('Bulan:',j)\n",
    "        temp_df_period=data_workshop.query('tahun==@i').query('bulan==@j')\n",
    "        for k in ['total_precipitation_0_daily','2m_temperature_0_daily','2m_dewpoint_temperature_0_daily',\n",
    "                  'convective_rain_rate_0_daily','potential_evaporation_0_daily','evaporation_0_daily',\n",
    "                  'soil_temperature_level_1_0_daily','volumetric_soil_water_layer_1_0_daily']:\n",
    "            print('Data:',k)\n",
    "            print('Daily Statistics:')\n",
    "            for l in ['mean','max','min']:\n",
    "                print(l,end ='|')\n",
    "                last_day=get_last_day_of_month(int('20'+str(i)),j)\n",
    "                last_date=f'20{i}-{str(j).zfill(2)}-{last_day}'\n",
    "                date_before=days_before(last_date,719)\n",
    "                ncfile=xr.open_dataset(f'/data/ksa/00_Code/12_Workshop_Puso/concated_climate_data/concated-ERA5-{k}-{l}.nc')\n",
    "                keys=dict_nc_value[k]\n",
    "                var_nc=f'{keys}_{l}'\n",
    "                temp_df_period[var_nc]=temp_df_period.apply(lambda y: ncfile[keys]\n",
    "                                                                   .sel(latitude=y['latitude'], \n",
    "                                                                        longitude=y['longitude'], \n",
    "                                                                        method='nearest')  # Select nearest latitude and longitude\n",
    "                                                                   .sel(valid_time=slice(date_before, last_date))  \n",
    "                                                                   .values, axis=1)\n",
    "                list_var=[f'{keys}_{l}_{59-o}' for o in range(0,60)]\n",
    "                temp_df_period[var_nc]=temp_df_period[var_nc].apply(lambda y:y.reshape(-1,12).mean(axis=1))#.values)\n",
    "                for m in range(0,60):\n",
    "                    temp_df_period[f'ERA5_{keys}_{l}_{(str(59-m).zfill(2))}']=temp_df_period[var_nc].apply(lambda y:y[m])\n",
    "                temp_df_period.drop(var_nc,axis=1,inplace=True)\n",
    "            print('\\n')\n",
    "        data_compile=pd.concat([temp_df_period,data_compile])\n",
    "        print('***************************')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17bf68fe-21fa-4285-b286-4ed683502589",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_compile.drop(['latitude','longitude'],axis=1).to_csv('data_compile_agro_ERA5 data.csv',sep=';',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}