Merge Peilbeheerst 2024 into main repository (#111)

Co-authored-by: Guy-HKV <[email protected]> Co-authored-by: rbruijnshkv <[email protected]> Co-authored-by: Martijn Visser <[email protected]>
Deltares · Jul 29, 2024 · 7b85771 · 7b85771
1 parent 21055ea
commit 7b85771
Show file tree

Hide file tree

Showing 117 changed files with 27,626 additions and 13,618 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,5 +20,18 @@ dmypy.json
 # pytest --basetemp
 src/hydamo/tests/temp/
 src/ribasim_nl/tests/temp/
+src/peilbeheerst_model/tests/temp/
+
+src/peilbeheerst_model/*.html
+src/peilbeheerst_model/*.ipynb
+src/peilbeheerst_model/*.code-workspace
+src/peilbeheerst_model/.vscode
+
+src/peilbeheerst_model/*.jpeg
+src/peilbeheerst_model/*.gpkg
+src/peilbeheerst_model/tests_results
+/src/peilbeheerst_model/Output_zdrive
+/src/peilbeheerst_model/Rekenend_Model_Test
+/src/peilbeheerst_model/vervallen
 
 notebooks/rijkswaterstaat/plots/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -14,14 +14,14 @@ repos:
         exclude: '.teamcity'
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.6
+    rev: v0.5.5
     hooks:
       - id: ruff
         types_or: [python, pyi, jupyter]
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
         types_or: [python, pyi, jupyter]
   - repo: https://github.com/kynan/nbstripout
-    rev: 0.6.1
+    rev: 0.7.1
     hooks:
       - id: nbstripout
diff --git a/app/main.py b/app/main.py
@@ -62,9 +62,7 @@ def add_lines(attr, old, new):
     for node_id in node_ids:
         for active_variable in time_fig_variables.active:
             variable_name = basin_variable_columns[active_variable]
-            df = basin_results_df[basin_results_df.node_id == node_id][
-                ["time", variable_name]
-            ]
+            df = basin_results_df[basin_results_df.node_id == node_id][["time", variable_name]]
             max_y += [df[variable_name].max()]
             min_y += [df[variable_name].min()]
             source = ColumnDataSource(df)
@@ -95,14 +93,11 @@ def update_map_values(attr, old, new):
 
     # slice results
     df_results_select = basin_results_df[
-        basin_results_df["time"]
-        == date_time_slider.value_as_datetime.date().isoformat()
+        basin_results_df["time"] == date_time_slider.value_as_datetime.date().isoformat()
     ].set_index("node_id")[variable]
 
     # update locations source
-    locations_source.data["value"] = df_select.node_id.apply(
-        lambda x: abs(df_results_select.at[x])
-    ).to_list()
+    locations_source.data["value"] = df_select.node_id.apply(lambda x: abs(df_results_select.at[x])).to_list()
 
     # update map_fig_variable_range
     map_fig_variable_range.start = 0
@@ -132,9 +127,7 @@ def update_cm(attr, old, new):
 # basin results
 basin_results_df = pd.read_feather(results_dir / "basin.arrow")
 basin_results_df.reset_index(inplace=True)
-basin_variable_columns = [
-    i for i in basin_results_df.columns if i not in ["time", "node_id"]
-]
+basin_variable_columns = [i for i in basin_results_df.columns if i not in ["time", "node_id"]]
 if INIT_VARIABLE in basin_variable_columns:
     actives = [basin_variable_columns.index(INIT_VARIABLE)]
 else:
@@ -197,9 +190,7 @@ def update_cm(attr, old, new):
 time_fig.xaxis.formatter = get_formatter()
 time_fig.xaxis.major_label_orientation = math.pi / 4
 
-time_line = Span(
-    location=model.starttime, dimension="height", line_color="red", line_width=3
-)
+time_line = Span(location=model.starttime, dimension="height", line_color="red", line_width=3)
 
 time_fig.add_layout(time_line)
 
@@ -238,9 +229,7 @@ def update_cm(attr, old, new):
         column(Div(text="<b>grafiek:</b>"), time_fig_variables),
     ),
 )
-control_layout = column(
-    variables_layout, map_fig_variable_range, date_time_slider.widget
-)
+control_layout = column(variables_layout, map_fig_variable_range, date_time_slider.widget)
 
 layout = row(map_fig, time_fig, control_layout)
 

diff --git a/app/widgets/date_time_slider.py b/app/widgets/date_time_slider.py
@@ -8,9 +8,7 @@
 
 
 def get_formatter(format="%Y-%m-%d %H:%M:%S"):
-    return DatetimeTickFormatter(
-        years=format, days=format, hours=format, minutes=format
-    )
+    return DatetimeTickFormatter(years=format, days=format, hours=format, minutes=format)
 
 
 def round_seconds(ts_seconds, td_seconds, method):
@@ -22,9 +20,7 @@ def round_seconds(ts_seconds, td_seconds, method):
     return rounder(ts_seconds / td_seconds) * td_seconds
 
 
-def round_datetime(
-    dt, timedelta_round, method: Literal["floor", "ceil", "round"] = "round"
-):
+def round_datetime(dt, timedelta_round, method: Literal["floor", "ceil", "round"] = "round"):
     ts_seconds = dt.timestamp()
     td_seconds = timedelta_round.total_seconds()
     rounded_seconds = round_seconds(ts_seconds, td_seconds, method)

diff --git a/app/widgets/map_figure.py b/app/widgets/map_figure.py
@@ -116,17 +116,13 @@ def make_options(
 ):
     # set overlay and handlers
     overlay_options = list(map_overlays.keys())
-    active_overlays = [
-        idx for idx, (_, v) in enumerate(map_overlays.items()) if v["visible"]
-    ]
+    active_overlays = [idx for idx, (_, v) in enumerate(map_overlays.items()) if v["visible"]]
     overlay_control = CheckboxGroup(labels=overlay_options, active=active_overlays)
     overlay_control.on_change("active", overlays_change)
 
     # set background and handlers
     background_options = list(BOKEH_BACKGROUNDS.keys())
-    background_active = list(BOKEH_BACKGROUNDS.keys()).index(
-        BOKEH_SETTINGS["background"]
-    )
+    background_active = list(BOKEH_BACKGROUNDS.keys()).index(BOKEH_SETTINGS["background"])
     background_control = RadioGroup(labels=background_options, active=background_active)
     background_control.on_change("active", background_change)
     map_controls = column(

diff --git a/notebooks/afleiding_hoofdwateren/klassificatie_waterlijnen.py b/notebooks/afleiding_hoofdwateren/klassificatie_waterlijnen.py
@@ -13,23 +13,17 @@
 }
 
 # Bestanden op de cloud-storage
-waterlijnen_rws = cloud.joinpath(
-    "Rijkswaterstaat", "aangeleverd", "BRT_waterlijnen_RWS", "WaterenNL.shp"
-)
+waterlijnen_rws = cloud.joinpath("Rijkswaterstaat", "aangeleverd", "BRT_waterlijnen_RWS", "WaterenNL.shp")
 recategorize_xlsx = cloud.joinpath("Rijkswaterstaat", "verwerkt", "recategorize.xlsx")
-result_gpkg = cloud.joinpath(
-    "Rijkswaterstaat", "Verwerkt", "categorie_oppervlaktewater.gpkg"
-)
+result_gpkg = cloud.joinpath("Rijkswaterstaat", "Verwerkt", "categorie_oppervlaktewater.gpkg")
 
 # %% Fix RWS-lijnen
 rws_waterlijn_gdf = gpd.read_file(waterlijnen_rws, engine="pyogrio", fid_as_index=True)
 rws_waterlijn_gdf.loc[:, "categorie"] = None
 
 # drop all FIDs we don't need. Mainly offshore.
 drop_fid_df = pd.read_excel(recategorize_xlsx, sheet_name="RWS_EXCLUDE_FID")
-rws_waterlijn_gdf = rws_waterlijn_gdf.loc[
-    ~rws_waterlijn_gdf.index.isin(drop_fid_df["FID"])
-]
+rws_waterlijn_gdf = rws_waterlijn_gdf.loc[~rws_waterlijn_gdf.index.isin(drop_fid_df["FID"])]
 
 # first map scales to water systems
 for k, v in MAP_SCHALE.items():
@@ -41,36 +35,28 @@
 rws_waterlijn_gdf = rws_waterlijn_gdf[rws_waterlijn_gdf["categorie"].notna()]
 
 # remap to national_hw
-label_to_national_hw_df = pd.read_excel(
-    recategorize_xlsx, sheet_name="RWS_LABEL_TO_NATIONAL_HW"
+label_to_national_hw_df = pd.read_excel(recategorize_xlsx, sheet_name="RWS_LABEL_TO_NATIONAL_HW")
+rws_waterlijn_gdf.loc[rws_waterlijn_gdf.Label.isin(label_to_national_hw_df["Label"]), ["categorie"]] = (
+    "nationaal hoofdwater"
 )
-rws_waterlijn_gdf.loc[
-    rws_waterlijn_gdf.Label.isin(label_to_national_hw_df["Label"]), ["categorie"]
-] = "nationaal hoofdwater"
 
 # remap to regional_hw
-label_to_regional_hw_df = pd.read_excel(
-    recategorize_xlsx, sheet_name="RWS_LABEL_TO_REGIONAL_HW"
+label_to_regional_hw_df = pd.read_excel(recategorize_xlsx, sheet_name="RWS_LABEL_TO_REGIONAL_HW")
+rws_waterlijn_gdf.loc[rws_waterlijn_gdf.Label.isin(label_to_regional_hw_df["Label"]), ["categorie"]] = (
+    "regionaal hoofdwater"
 )
-rws_waterlijn_gdf.loc[
-    rws_waterlijn_gdf.Label.isin(label_to_regional_hw_df["Label"]), ["categorie"]
-] = "regionaal hoofdwater"
 
-fid_to_regional_hw_df = pd.read_excel(
-    recategorize_xlsx, sheet_name="RWS_FID_TO_REGIONAL_HW"
+fid_to_regional_hw_df = pd.read_excel(recategorize_xlsx, sheet_name="RWS_FID_TO_REGIONAL_HW")
+
+rws_waterlijn_gdf.loc[rws_waterlijn_gdf.index.isin(fid_to_regional_hw_df["FID"]), ["categorie"]] = (
+    "regionaal hoofdwater"
 )
 
-rws_waterlijn_gdf.loc[
-    rws_waterlijn_gdf.index.isin(fid_to_regional_hw_df["FID"]), ["categorie"]
-] = "regionaal hoofdwater"
+fid_to_national_hw_df = pd.read_excel(recategorize_xlsx, sheet_name="RWS_FID_TO_NATIONAL_HW")
 
-fid_to_national_hw_df = pd.read_excel(
-    recategorize_xlsx, sheet_name="RWS_FID_TO_NATIONAL_HW"
+rws_waterlijn_gdf.loc[rws_waterlijn_gdf.index.isin(fid_to_national_hw_df["FID"]), ["categorie"]] = (
+    "nationaal hoofdwater"
 )
 
-rws_waterlijn_gdf.loc[
-    rws_waterlijn_gdf.index.isin(fid_to_national_hw_df["FID"]), ["categorie"]
-] = "nationaal hoofdwater"
-
 
 rws_waterlijn_gdf.to_file(result_gpkg, layer="waterlijnen_rws", engine="pyogrio")
diff --git a/notebooks/afleiding_hoofdwateren/nationaal_hoofdwater.py b/notebooks/afleiding_hoofdwateren/nationaal_hoofdwater.py
@@ -31,23 +31,15 @@
 top10_nl_water = cloud.joinpath("Basisgegevens", "Top10NL", "top10nl_Compleet.gpkg")
 
 
-watervlak_gpkg = cloud.joinpath(
-    "Rijkswaterstaat", "Verwerkt", "categorie_oppervlaktewater.gpkg"
-)
+watervlak_gpkg = cloud.joinpath("Rijkswaterstaat", "Verwerkt", "categorie_oppervlaktewater.gpkg")
 
 
-result_gpkg = cloud.joinpath(
-    "Rijkswaterstaat", "Verwerkt", "categorie_oppervlaktewater.gpkg"
-)
+result_gpkg = cloud.joinpath("Rijkswaterstaat", "Verwerkt", "categorie_oppervlaktewater.gpkg")
 
 print("read")
-watervlak_gdf = gpd.read_file(
-    top10_nl_water, layer="top10nl_waterdeel_vlak", engine="pyogrio", fid_as_index=True
-)
+watervlak_gdf = gpd.read_file(top10_nl_water, layer="top10nl_waterdeel_vlak", engine="pyogrio", fid_as_index=True)
 
-rws_waterlijn_gdf = gpd.read_file(
-    result_gpkg, layer="waterlijnen_rws", engine="pyogrio", fid_as_index=True
-)
+rws_waterlijn_gdf = gpd.read_file(result_gpkg, layer="waterlijnen_rws", engine="pyogrio", fid_as_index=True)
 
 recategorize_xlsx = cloud.joinpath("Rijkswaterstaat", "verwerkt", "recategorize.xlsx")
 
@@ -57,32 +49,22 @@
     # merge all name-categories
     print("merge name-fields")
     watervlak_gdf.loc[:, ["naam"]] = watervlak_gdf["naamofficieel"]
-    watervlak_gdf.loc[watervlak_gdf.naam.isna(), ["naam"]] = watervlak_gdf[
-        watervlak_gdf.naam.isna()
-    ]["naamnl"]
-    watervlak_gdf.loc[watervlak_gdf.naam.isna(), ["naam"]] = watervlak_gdf[
-        watervlak_gdf.naam.isna()
-    ]["naamfries"]
+    watervlak_gdf.loc[watervlak_gdf.naam.isna(), ["naam"]] = watervlak_gdf[watervlak_gdf.naam.isna()]["naamnl"]
+    watervlak_gdf.loc[watervlak_gdf.naam.isna(), ["naam"]] = watervlak_gdf[watervlak_gdf.naam.isna()]["naamfries"]
 
     print("name sluizen")
     for k, v in MAP_SLUIZEN.items():
         watervlak_gdf.loc[watervlak_gdf.sluisnaam == k, ["naam"]] = v
 
-    exclude_names_df = pd.read_excel(
-        recategorize_xlsx, sheet_name="TOP10NL_EXCLUDE_NAMES"
-    )
+    exclude_names_df = pd.read_excel(recategorize_xlsx, sheet_name="TOP10NL_EXCLUDE_NAMES")
 
-    watervlak_gdf = watervlak_gdf.loc[
-        ~watervlak_gdf.naam.isin(exclude_names_df["naam"])
-    ]
+    watervlak_gdf = watervlak_gdf.loc[~watervlak_gdf.naam.isin(exclude_names_df["naam"])]
 
     print("dissolve to naam, fysiekvoorkomen and hoofdafwatering")
     data = {"naam": [], "geometry": [], "fysiekvoorkomen": []}
 
     # fill fysiekvoorkomen so polygons don't get dropped
-    watervlak_gdf.loc[
-        watervlak_gdf["fysiekvoorkomen"].isna(), "fysiekvoorkomen"
-    ] = "overig"
+    watervlak_gdf.loc[watervlak_gdf["fysiekvoorkomen"].isna(), "fysiekvoorkomen"] = "overig"
 
     # custom dissolve section. We dissolve only adjacent polygons with the same name and not over sluices, bridges etc (fysiekvoorkomen)
     for (name, voorkomen), df in watervlak_gdf.groupby(by=["naam", "fysiekvoorkomen"]):
@@ -102,9 +84,7 @@
     watervlak_diss_gdf = pd.concat(
         [
             gpd.GeoDataFrame(data, crs=28992),
-            watervlak_gdf[watervlak_gdf.naam.isna()][
-                ["naam", "fysiekvoorkomen", "geometry"]
-            ],
+            watervlak_gdf[watervlak_gdf.naam.isna()][["naam", "fysiekvoorkomen", "geometry"]],
         ]
     ).reset_index(drop=True)
 
@@ -117,9 +97,7 @@
     watervlak_diss_gdf.to_file(result_gpkg, layer="watervlak", engine="pyogrio")
 else:
     print("read")
-    watervlak_diss_gdf = gpd.read_file(
-        result_gpkg, layer="watervlak", engine="pyogrio", fid_as_index=True
-    )
+    watervlak_diss_gdf = gpd.read_file(result_gpkg, layer="watervlak", engine="pyogrio", fid_as_index=True)
     # reset categorie column
     watervlak_diss_gdf.loc[:, ["categorie"]] = None
 
@@ -147,15 +125,11 @@
 for row in waterlijn_select_gdf.itertuples():
     # get row indexes by spatial index and geometric selection
     idx = watervlak_diss_gdf.sindex.intersection(row.geometry.bounds)
-    idx = watervlak_diss_gdf.iloc[idx][
-        watervlak_diss_gdf.iloc[idx].intersects(row.geometry)
-    ].index
+    idx = watervlak_diss_gdf.iloc[idx][watervlak_diss_gdf.iloc[idx].intersects(row.geometry)].index
 
     if row.Label not in ignore_min_length:
         # should have sigificant overlap with polygon
-        mask = watervlak_diss_gdf.loc[idx].intersection(row.geometry).length > (
-            row.geometry.length * overlap
-        )
+        mask = watervlak_diss_gdf.loc[idx].intersection(row.geometry).length > (row.geometry.length * overlap)
 
         # or not being "overig" (sluis, bridge, etc)
         mask = mask | (watervlak_diss_gdf.loc[idx].fysiekvoorkomen != "overig")
@@ -171,9 +145,7 @@
 
     # give rws-label if Top10NL name is None
     if row.Bron == "NWB Vaarwegen":
-        name_idx = watervlak_diss_gdf.loc[idx][
-            watervlak_diss_gdf.loc[idx].naam.isna()
-        ].index
+        name_idx = watervlak_diss_gdf.loc[idx][watervlak_diss_gdf.loc[idx].naam.isna()].index
         watervlak_diss_gdf.loc[name_idx, ["naam"]] = row.Label
 
 # Recategorize for
@@ -186,13 +158,9 @@
 watervlak_diss_gdf.loc[watervlak_diss_gdf.index.isin(reset_fids), ["categorie"]] = None
 watervlak_diss_gdf.loc[watervlak_diss_gdf.naam.isin(reset_names), ["categorie"]] = None
 
-to_nationaal_hw_df = pd.read_excel(
-    recategorize_xlsx, sheet_name="TOP10NL_TO_NATIONAL_HW"
-)
+to_nationaal_hw_df = pd.read_excel(recategorize_xlsx, sheet_name="TOP10NL_TO_NATIONAL_HW")
 
-watervlak_diss_gdf.loc[
-    watervlak_diss_gdf.naam.isin(to_nationaal_hw_df["naam"]), ["categorie"]
-] = "nationaal hoofdwater"
+watervlak_diss_gdf.loc[watervlak_diss_gdf.naam.isin(to_nationaal_hw_df["naam"]), ["categorie"]] = "nationaal hoofdwater"
 
 # %% Write to disk
 print("write")

diff --git a/notebooks/afleiding_hoofdwateren/oppervlaktewater_belgie.py b/notebooks/afleiding_hoofdwateren/oppervlaktewater_belgie.py
@@ -7,26 +7,18 @@
 
 basins_osm_gpkg = cloud.joinpath("Basisgegevens", "OSM", "Belgie", "water_areas.gpkg")
 
-basins_user_data_gpkg = cloud.joinpath(
-    "Rijkswaterstaat", "verwerkt", "basins_user_data.gpkg"
-)
+basins_user_data_gpkg = cloud.joinpath("Rijkswaterstaat", "verwerkt", "basins_user_data.gpkg")
 
-rivers_osm_gpkg = cloud.joinpath(
-    "Basisgegevens", "OSM", "Nederland_Belgie", "waterway_river.gpkg"
-)
-canal_osm_gpkg = cloud.joinpath(
-    "Basisgegevens", "OSM", "Nederland_Belgie", "waterway_canal.gpkg"
-)
+rivers_osm_gpkg = cloud.joinpath("Basisgegevens", "OSM", "Nederland_Belgie", "waterway_river.gpkg")
+canal_osm_gpkg = cloud.joinpath("Basisgegevens", "OSM", "Nederland_Belgie", "waterway_canal.gpkg")
 
 # %%
 clip_poly = gpd.read_file(
     basins_user_data_gpkg, layer="clip_osm_basins", engine="pyogrio", fid_as_index=True
 ).unary_union
 
 basins_osm_gdf = (
-    gpd.read_file(basins_osm_gpkg, engine="pyogrio", fid_as_index=True)
-    .clip(clip_poly)
-    .explode(ignore_index=True)
+    gpd.read_file(basins_osm_gpkg, engine="pyogrio", fid_as_index=True).clip(clip_poly).explode(ignore_index=True)
 )
 
 canal_osm_gdf = gpd.read_file(canal_osm_gpkg, engine="pyogrio", fid_as_index=True)