Skip to content

Commit

Permalink
Merge pull request #27 from Vizzuality/data/pecalculations
Browse files Browse the repository at this point in the history 
Create animated tiles
  • Loading branch information
@ikerey
ikerey authored Nov 14, 2024
2 parents dc604ca + 99e4c54 commit f8fdfc6
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 38 deletions.
192 changes: 161 additions & 31 deletions data-processing/notebooks/03_create_layers.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,21 +102,21 @@
"name": "stdout",
"output_type": "stream",
"text": [
"{'Agricultural drought exposure': <datasets.datasets.Dataset object at 0x7fc8f0293a70>,\n",
" 'Agricultural drought hazard': <datasets.datasets.Dataset object at 0x7fc8f0293b00>,\n",
" 'Boundaries': <datasets.datasets.Dataset object at 0x7fc8f0293ad0>,\n",
" 'Contextual layers': <datasets.datasets.Dataset object at 0x7fc8f0293aa0>,\n",
" 'EO-based flood exposure': <datasets.datasets.Dataset object at 0x7fc8f0293a40>,\n",
" 'EO-based flood hazard': <datasets.datasets.Dataset object at 0x7fc8f0293a10>,\n",
" 'Hydrographic data': <datasets.datasets.Dataset object at 0x7fc8f0293830>,\n",
" 'Hydrometeorological Data': <datasets.datasets.Dataset object at 0x7fc8f02939b0>,\n",
" 'Meteorological drought exposure': <datasets.datasets.Dataset object at 0x7fc8f02939e0>,\n",
" 'Meteorological drought hazard': <datasets.datasets.Dataset object at 0x7fc8f0293980>,\n",
" 'Model-based flood exposure': <datasets.datasets.Dataset object at 0x7fc8f0293950>,\n",
" 'Model-based flood hazard': <datasets.datasets.Dataset object at 0x7fc8f0293920>,\n",
" 'Populated infrastructures': <datasets.datasets.Dataset object at 0x7fc8f02938f0>,\n",
" 'Transportation Network Infrastructures': <datasets.datasets.Dataset object at 0x7fc8f02938c0>,\n",
" 'Water-related infrastructures': <datasets.datasets.Dataset object at 0x7fc8f0293860>}\n"
"{'Agricultural drought exposure': <datasets.datasets.Dataset object at 0x7f0c17fb3920>,\n",
" 'Agricultural drought hazard': <datasets.datasets.Dataset object at 0x7f0c17fb38f0>,\n",
" 'Boundaries': <datasets.datasets.Dataset object at 0x7f0c17fb3890>,\n",
" 'Contextual layers': <datasets.datasets.Dataset object at 0x7f0c17fb3860>,\n",
" 'EO-based flood exposure': <datasets.datasets.Dataset object at 0x7f0c17fb3680>,\n",
" 'EO-based flood hazard': <datasets.datasets.Dataset object at 0x7f0c17fb3800>,\n",
" 'Hydrographic data': <datasets.datasets.Dataset object at 0x7f0c17fb3830>,\n",
" 'Hydrometeorological Data': <datasets.datasets.Dataset object at 0x7f0c17fb37d0>,\n",
" 'Meteorological drought exposure': <datasets.datasets.Dataset object at 0x7f0c17fb37a0>,\n",
" 'Meteorological drought hazard': <datasets.datasets.Dataset object at 0x7f0c17fb3770>,\n",
" 'Model-based flood exposure': <datasets.datasets.Dataset object at 0x7f0c17fb3740>,\n",
" 'Model-based flood hazard': <datasets.datasets.Dataset object at 0x7f0c17fb3710>,\n",
" 'Populated infrastructures': <datasets.datasets.Dataset object at 0x7f0c17fb36b0>,\n",
" 'Transportation Network Infrastructures': <datasets.datasets.Dataset object at 0x7f0c17fb36e0>,\n",
" 'Water-related infrastructures': <datasets.datasets.Dataset object at 0x7f0c17fb3620>}\n"
]
}
],
Expand Down Expand Up @@ -1048,7 +1048,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Processing year 2023\n"
"Processing year 2019\n"
]
},
{
Expand All @@ -1074,13 +1074,87 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Processing Soil moisture from Hydrometeorological Data\n",
"Loading Zarr data from gs://wbhydross_deliverables/D3-Database/03-Soil Moisture_hydrological_soil_properties/Soil Moisture-ERA5Land/WBHYDROSSD_ERA5-Land_mean-swvl1swvl2-M-mean_South_Sudan-EPSG4326_195001_202312_20240208154611.zarr...\n",
"Processing Temperature from Hydrometeorological Data\n",
"Loading Zarr data from gs://wbhydross_deliverables/D3-Database/02- Meteorological datasets/Temperature-ERA5Land/WBHYDROSSD_ERA5-Land_t2m-D-mean_South_Sudan-EPSG4326_19500101_20231231_20240209091716.zarr...\n",
"Creating tiles ...\n",
"[########################################] | 100% Completed | 34m 33s\n",
"[########################################] | 100% Completed | 34m 27s\n",
"Creating APNGs\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 1/1 [34:29<00:00, 2069.53s/it]\n",
"100%|██████████| 1/1 [34:29<00:00, 2069.53s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing year 2020\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0%| | 0/1 [00:00<?, ?it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Hydrometeorological Data\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": []
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing Temperature from Hydrometeorological Data\n",
"Loading Zarr data from gs://wbhydross_deliverables/D3-Database/02- Meteorological datasets/Temperature-ERA5Land/WBHYDROSSD_ERA5-Land_t2m-D-mean_South_Sudan-EPSG4326_19500101_20231231_20240209091716.zarr...\n",
"Creating tiles ...\n",
"[########################################] | 100% Completed | 35m 7ss\n",
"Creating APNGs\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 1/1 [35:09<00:00, 2109.77s/it]\n",
"100%|██████████| 1/1 [35:09<00:00, 2109.78s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing year 2021\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0%| | 0/1 [00:00<?, ?it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Hydrometeorological Data\n"
]
},
{
"name": "stderr",
"output_type": "stream",
Expand All @@ -1093,16 +1167,16 @@
"Processing Temperature from Hydrometeorological Data\n",
"Loading Zarr data from gs://wbhydross_deliverables/D3-Database/02- Meteorological datasets/Temperature-ERA5Land/WBHYDROSSD_ERA5-Land_t2m-D-mean_South_Sudan-EPSG4326_19500101_20231231_20240209091716.zarr...\n",
"Creating tiles ...\n",
"[########################################] | 100% Completed | 34m 17s\n",
"[########################################] | 100% Completed | 35m 16s\n",
"Creating APNGs\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 2/2 [1:08:55<00:00, 2067.88s/it]\n",
"100%|██████████| 1/1 [1:08:55<00:00, 4135.76s/it]\n"
"100%|██████████| 1/1 [35:18<00:00, 2118.96s/it]\n",
"100%|██████████| 1/1 [35:18<00:00, 2118.96s/it]\n"
]
}
],
Expand All @@ -1118,7 +1192,7 @@
"\n",
"dict_path = \"../data/processed/datasets_dict.json\"\n",
"\n",
"for year in range(2023, 2024):\n",
"for year in range(2019, 2022):\n",
" print(f\"Processing year {year}\")\n",
" layer_processing = LayerProcessing(datasets, datasets_list, dict_path)\n",
" layer_processing.create_layers(min_z=4, max_z=10, year=year)"
Expand All @@ -1133,54 +1207,110 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Uploading folder HD_precipitation:\n"
"Uploading folder 8:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 80/80 [00:02<00:00, 32.99it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Uploading folder 10:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 1073/1073 [00:07<00:00, 142.60it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Uploading folder 5:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 4/4 [00:01<00:00, 3.40it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Uploading folder 9:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 285/285 [00:02<00:00, 100.44it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Uploading folder 6:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 33856/33856 [06:17<00:00, 89.64it/s] \n"
"100%|██████████| 9/9 [00:01<00:00, 8.72it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Uploading folder HD_temperature:\n"
"Uploading folder 4:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 35328/35328 [05:06<00:00, 115.42it/s]\n"
"100%|██████████| 1/1 [00:01<00:00, 1.47s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Uploading folder HD_soil_moisture:\n"
"Uploading folder 7:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 51520/51520 [07:39<00:00, 112.02it/s]\n"
"100%|██████████| 20/20 [00:01<00:00, 12.11it/s]\n"
]
}
],
"source": [
"directory_path = \"../data/processed/AnimatedTiles/\"\n",
"directory_path = \"../data/processed/AnimatedTiles\"\n",
"bucket_folder = \"animated-tiles\"\n",
"\n",
"all_folders = os.listdir(directory_path)\n",
Expand Down
10 changes: 4 additions & 6 deletions data-processing/src/animations/animated_tiles.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import io
import os
import warnings
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

Expand All @@ -12,7 +13,6 @@
import numpy as np
import rasterio
import xarray as xr
import warnings
from dask.diagnostics import ProgressBar
from PIL import Image
from rio_tiler.colormap import ColorMapType
Expand All @@ -24,6 +24,7 @@
# Suppress specific warnings from rasterio
warnings.filterwarnings("ignore", category=rasterio.errors.NotGeoreferencedWarning)


class AnimatedTiles:
"""
Class for creating animated tiles.
Expand Down Expand Up @@ -289,7 +290,7 @@ def _worker_create_tiles(self, da, n, tiles):
n,
self.color_map,
)

def _get_slice_data(self, time, time_coord="time"):
"""Slice the raster dataset based on the time coordinate."""
da = self.data.isel({time_coord: time}).copy()
Expand All @@ -308,12 +309,9 @@ def generate_tiles(self, time_coord="time"):
tiles = list(mercantile.tiles(bbox[0], bbox[1], bbox[2], bbox[3], zooms=self.zooms))

tasks = [
dask.delayed(self._worker_create_tiles)(
self._get_slice_data(n, time_coord), n, tiles
)
dask.delayed(self._worker_create_tiles)(self._get_slice_data(n, time_coord), n, tiles)
for n in range(len(time_coords))
]

with ProgressBar(minimum=0.01):
dask.compute(*tasks)

2 changes: 1 addition & 1 deletion data-processing/src/datasets/datasets_config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@
"type": "raster",
"format": "Zarr",
"base_url": "gs://wbhydross_deliverables/D3-Database/02- Meteorological datasets/Temperature-ERA5Land/WBHYDROSSD_ERA5-Land_t2m-D-mean_South_Sudan-EPSG4326_19500101_20231231_20240209091716.zarr",
"styles": {"colors": ["#FFFFFF", "#E8eB5B", "#211F4D"], "vmin": 10, "vmax": 35, "time_coord": "month"}
"styles": {"colors": ["#FFFFFF", "#E85B5B", "#211F4D"], "vmin": 10, "vmax": 35, "time_coord": "month"}
},
"Evapotranspiration": {
"type": "raster",
Expand Down

0 comments on commit f8fdfc6

Please sign in to comment.