Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Run post-process_agv.py #172

Merged
merged 3 commits into from
Oct 21, 2024
Merged

Run post-process_agv.py #172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
56eb3be
Run post-process_agv.py
visr Oct 21, 2024
8fa2144
Remove a general_functions.py copy
visr Oct 21, 2024
a6908e4
Read all layers from gpkg
visr Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions src/peilbeheerst_model/peilbeheerst_model/crossings_to_ribasim.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,13 @@ def __init__(self, model_characteristics):

def read_gpkg_layers(
self,
variables=["hydroobject", "gemaal", "stuw", "peilgebied", "streefpeil", "duikersifonhevel"],
print_var=False,
data=None,
):
"""Read specified layers from a GeoPackage file and store them in the object.

Parameters
----------
variables : list, optional
List of layer names to be read from the GeoPackage, by default
["hydroobject", "gemaal", "stuw", "peilgebied", "streefpeil", "aggregation_area", 'duikersifonhevel']
print_var : bool, optional
Flag to print each layer name when reading, by default False
data : _type_, optional
Expand All @@ -69,11 +65,13 @@ def read_gpkg_layers(
"""
if data is None:
data = {}
for variable in variables:
gpkg_path = self.model_characteristics["path_postprocessed_data"]
layers = gpd.list_layers(gpkg_path)
for layer in layers.name:
if print_var:
print(variable)
data_temp = gpd.read_file(self.model_characteristics["path_postprocessed_data"], layer=variable)
data[variable] = data_temp
print(layer)
data_temp = gpd.read_file(gpkg_path, layer=layer)
data[layer] = data_temp

return data

Expand Down
12 changes: 6 additions & 6 deletions ...del/postprocess_data/general_functions.py → ...l/peilbeheerst_model/general_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,13 @@
import pandas as pd


def read_gpkg_layers(gpkg_path, variables, engine="fiona", print_var=False):
def read_gpkg_layers(gpkg_path, engine="fiona", print_var=False):
"""
Read specified layers from a GeoPackage (GPKG) file and return them as a dictionary.

Parameters
----------
gpkg_path (str): The file path to the GeoPackage (GPKG) file to read from.
variables (list): A list of layer names to read from the GeoPackage.
print_var (bool, optional): If True, print the name of each variable as it is read. Default is False.

Returns
Expand All @@ -22,11 +21,12 @@ def read_gpkg_layers(gpkg_path, variables, engine="fiona", print_var=False):
choose to print the names of variables as they are read by setting `print_var` to True.
"""
data = {}
for variable in variables:
layers = gpd.list_layers(gpkg_path)
for layer in layers.name:
if print_var:
print(variable)
data_temp = gpd.read_file(gpkg_path, layer=variable, engine=engine)
data[variable] = data_temp
print(layer)
data_temp = gpd.read_file(gpkg_path, layer=layer, engine=engine)
data[layer] = data_temp

return data

Expand Down
16 changes: 3 additions & 13 deletions src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_WSRL.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

import geopandas as gpd
import numpy as np
from general_functions import *

from peilbeheerst_model.general_functions import *

remove_cat_2 = True

Expand All @@ -35,18 +36,7 @@


# Load HHNK files
WSRL = read_gpkg_layers(
gpkg_path=data_path,
variables=[
"stuw",
"gemaal",
"hydroobject",
"duikersifonhevel",
"peilgebied",
"streefpeil",
"aggregation_area",
],
)
WSRL = read_gpkg_layers(gpkg_path=data_path)
WSRL["peilgebied"] = WSRL["peilgebied"].to_crs("EPSG:28992")

# Load waterschap boundaries
Expand Down
115 changes: 9 additions & 106 deletions src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_agv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
import geopandas as gpd
import numpy as np
import pandas as pd
from general_functions import read_gpkg_layers

from peilbeheerst_model.general_functions import read_gpkg_layers
from ribasim_nl import CloudStorage

# %%
Expand All @@ -23,9 +23,8 @@
# %%
cloud = CloudStorage()
cloud.download_verwerkt(waterschap)

# cloud.download_basisgegevens()
# cloud.download_aangeleverd("Rijkswaterstaat")
cloud.download_verwerkt("Rijkswaterstaat")
cloud.download_basisgegevens()

# %%
verwerkt_dir = cloud.joinpath(waterschap, "verwerkt")
Expand All @@ -37,27 +36,9 @@
# Hoofdwatersysteem boundaries
hws_path = cloud.joinpath("Rijkswaterstaat/verwerkt/krw_basins_vlakken.gpkg")

# Buffer boundaries
buffer_path = cloud.joinpath("Rijkswaterstaat/verwerkt/hws_buffer_agv.gpkg")

# Buffer RWHS
rhws_path = cloud.joinpath("Rijkswaterstaat/verwerkt/agv_rhws_buffer.gpkg")


# %% Load Files
# Load HHNK files
AVG = read_gpkg_layers(
gpkg_path=data_path,
variables=[
"stuw",
"gemaal",
"hydroobject",
"duikersifonhevel",
"peilgebied",
"streefpeil",
"aggregation_area",
],
)
AVG = read_gpkg_layers(gpkg_path=data_path)
AVG["peilgebied"] = AVG["peilgebied"].to_crs("EPSG:28992")

# Load waterschap boundaries
Expand All @@ -66,18 +47,8 @@
# Load hws
gdf_hws = gpd.read_file(hws_path)

# Load buffer
gdf_buffer = gpd.read_file(buffer_path)
gdf_buffer = gdf_buffer.to_crs("EPSG:28992")
gdf_buffer = gdf_buffer.dissolve()

# Load rhws
# gdf_rhws = gpd.read_file(rhws_path)
# gdf_rhws = gdf_rhws.to_crs('EPSG:28992')
# gdf_rhws = gdf_rhws.dissolve()

# %%
AVG["peilgebied"].globalid.is_unique
assert AVG["peilgebied"].globalid.is_unique

# Select waterschap boundaries and clip hws layer

Expand All @@ -89,9 +60,6 @@
# Use waterschap boundaries to clip HWS layer
gdf_hws = gpd.overlay(gdf_grens, gdf_hws, how="intersection")

# Use waterschap boundaries to clip HWS layer
# gdf_rhws = gpd.overlay(gdf_grens, gdf_rhws, how='intersection')

# Peilgebied and HWS layer overlap:
# 1. Identify the overlapping areas
# 2. Clip
Expand Down Expand Up @@ -122,42 +90,16 @@
peilgebieden_cat = []

for index, row in AVG["peilgebied"].iterrows():
# if row.code == "Oosterpark" or row.code == "Vechtboezem":
if "Oosterpark" in row.code or "Vechtboezem" in row.code or "Stadsboezem Amsterdam" in row.code:
print("true")

print(f"true {row.code}")
peilgebieden_cat.append(1)
else:
peilgebieden_cat.append(0)

# Add new column and drop old HWS_BZM column
AVG["peilgebied"]["peilgebied_cat"] = peilgebieden_cat

# %% Add rhws to ['peilgebied','streefpeil']
# update peilgebied dict key
# gdf_rhws['globalid'] = 'dummy_globalid_rhws_' + gdf_rhws.index.astype(str)
# gdf_rhws['code'] = 'dummy_code_nhws_' + gdf_rhws.index.astype(str)
# gdf_rhws['nen3610id'] = 'dummy_nen3610id_rhws_' + gdf_rhws.index.astype(str)
# gdf_rhws['peilgebied_cat'] = 1

# gdf_rhws = gdf_rhws[['globalid', 'code', 'nen3610id', 'peilgebied_cat', 'geometry']]

# AVG['peilgebied'] = pd.concat([gdf_rhws, AVG['peilgebied']])

# %%
# # Create boezem streefpeil layer
# streefpeil_hws = pd.DataFrame()
# streefpeil_hws['waterhoogte'] = [np.nan] * len(gdf_rhws)
# streefpeil_hws['globalid'] = 'dummy_globalid_rhws_' + gdf_rhws.index.astype(str)
# streefpeil_hws['geometry'] = [None]* len(gdf_rhws)

# AVG['streefpeil'] = pd.concat([streefpeil_hws, AVG['streefpeil']])
# AVG['streefpeil'] = gpd.GeoDataFrame(AVG['streefpeil'])

# Add nhws to ['peilgebied','streefpeil']

# %%
# update peilgebied dict key
# %% update peilgebied dict key
gdf_hws["globalid"] = "dummy_globalid_nhws_" + gdf_hws.index.astype(str)
gdf_hws["code"] = "dummy_code_nhws_" + gdf_hws.index.astype(str)
gdf_hws["nen3610id"] = "dummy_nen3610id_nhws_" + gdf_hws.index.astype(str)
Expand All @@ -167,8 +109,7 @@

AVG["peilgebied"] = pd.concat([gdf_hws, AVG["peilgebied"]])

# %%
# Create boezem streefpeil layer
# %% Create boezem streefpeil layer
streefpeil_hws = pd.DataFrame()
streefpeil_hws["waterhoogte"] = [np.nan] * len(gdf_hws)
streefpeil_hws["globalid"] = "dummy_globalid_nhws_" + gdf_hws.index.astype(str)
Expand All @@ -177,44 +118,6 @@
AVG["streefpeil"] = pd.concat([streefpeil_hws, AVG["streefpeil"]])
AVG["streefpeil"] = gpd.GeoDataFrame(AVG["streefpeil"])

# %% Create buffer polygon between NHWS and peilgebied/RHWS
# buffer_polygon = gdf_buffer.geometry.iat[0].intersection(gdf_grens.geometry.iat[0])
# buffer_polygon = buffer_polygon.difference(shapely.geometry.MultiPolygon(gdf_hws.geometry.tolist()))
# buffer_polygon = buffer_polygon.difference(shapely.ops.unary_union(AVG['peilgebied'].geometry.tolist()))

# buffer_polygon = gpd.GeoDataFrame(buffer_polygon)
# buffer_polygon = buffer_polygon.set_geometry(0)
# buffer_polygon = buffer_polygon.dissolve()
# buffer_polygon = buffer_polygon.rename(columns={0:'geometry'})
# buffer_polygon = buffer_polygon.set_geometry('geometry')
# buffer_polygon = buffer_polygon.set_crs('EPSG:28992')


# %% Add buffer to ['peilgebied','streefpeil']

# update peilgebied dict key
# buffer_polygon = gpd.GeoDataFrame(buffer_polygon)
# buffer_polygon['globalid'] = 'dummy_globalid_nhws_buffer_' + buffer_polygon.index.astype(str)
# buffer_polygon['code'] = 'dummy_code_nhws_buffer_' + buffer_polygon.index.astype(str)
# buffer_polygon['nen3610id'] = 'dummy_nen3610id_nhws_buffer_' + buffer_polygon.index.astype(str)
# buffer_polygon['peilgebied_cat'] = 2
# buffer_polygon = buffer_polygon.rename(columns={0:'geometry'})
# buffer_polygon = buffer_polygon[['globalid', 'code', 'nen3610id', 'peilgebied_cat', 'geometry']]

# AVG['peilgebied'] = pd.concat([buffer_polygon, AVG['peilgebied']])
# AVG['peilgebied'] = gpd.GeoDataFrame(AVG['peilgebied'])

# %%
# # Create boezem streefpeil layer
# streefpeil_buffer = pd.DataFrame()
# streefpeil_buffer['waterhoogte'] = [np.nan]
# streefpeil_buffer['globalid'] = 'dummy_globalid_nhws_buffer_' + buffer_polygon.index.astype(str)
# streefpeil_buffer['geometry'] = [None]


# AVG['streefpeil'] = pd.concat([streefpeil_buffer, AVG['streefpeil']])
# AVG['streefpeil'] = gpd.GeoDataFrame(AVG['streefpeil'])

# %%
if remove_cat_2:
AVG["peilgebied"] = AVG["peilgebied"].loc[AVG["peilgebied"].peilgebied_cat != 2]
Expand All @@ -227,6 +130,6 @@
print(key)
AVG[str(key)].to_file(output_gpkg_path, layer=str(key), driver="GPKG")

cloud.upload_verwerkt(output_gpkg_path)
cloud.upload_verwerkt(waterschap)
# %%
AVG["peilgebied"]["peilgebied_cat"].unique()
16 changes: 3 additions & 13 deletions src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_delfland.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@

import geopandas as gpd
import numpy as np
from general_functions import *

from peilbeheerst_model.general_functions import *

remove_cat_2 = True

Expand All @@ -36,18 +37,7 @@


# Load HHNK files
delfland = read_gpkg_layers(
gpkg_path=data_path,
variables=[
"stuw",
"gemaal",
"hydroobject",
"duikersifonhevel",
"peilgebied",
"streefpeil",
"aggregation_area",
],
)
delfland = read_gpkg_layers(gpkg_path=data_path)

delfland["peilgebied"] = delfland["peilgebied"].to_crs("EPSG:28992")

Expand Down
16 changes: 3 additions & 13 deletions src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_rijnland.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@

import geopandas as gpd
import numpy as np
from general_functions import *

from peilbeheerst_model.general_functions import *

remove_cat_2 = True

Expand All @@ -37,18 +38,7 @@


# Load HHNK files
Rijnland = read_gpkg_layers(
gpkg_path=data_path,
variables=[
"stuw",
"gemaal",
"hydroobject",
"duikersifonhevel",
"peilgebied",
"streefpeil",
"aggregation_area",
],
)
Rijnland = read_gpkg_layers(gpkg_path=data_path)
Rijnland["peilgebied"] = Rijnland["peilgebied"].to_crs("EPSG:28992")

# Load waterschap boundaries
Expand Down
16 changes: 3 additions & 13 deletions src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_wetterskip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

import geopandas as gpd
import numpy as np
from general_functions import *

from peilbeheerst_model.general_functions import *

remove_cat_2 = True

Expand Down Expand Up @@ -36,18 +37,7 @@


# Load HHNK files
Wetterskip = read_gpkg_layers(
gpkg_path=data_path,
variables=[
"stuw",
"gemaal",
"hydroobject",
"duikersifonhevel",
"peilgebied",
"streefpeil",
"aggregation_area",
],
)
Wetterskip = read_gpkg_layers(gpkg_path=data_path)
Wetterskip["peilgebied"] = Wetterskip["peilgebied"].to_crs("EPSG:28992")

# Load waterschap boundaries
Expand Down
Loading
Loading