Rws network (#40)

- misc fixes for building RWS network
- upload models to cloud-storage
@visr; can you pls review the cloud-part so others can start uploading
models?

I suggest not to delete this branch as there will be more updates...

---------

Co-authored-by: ngoorden <[email protected]>

docs/cloudstorage.qmd

@@ -61,3 +61,11 @@ authority = "Rijkswaterstaat"
 # upload local `aangeleverd` directory to cloud-storage
 cloud_storage.upload_aangeleverd(authority, overwrite=True)
 ```
+
+## Upload models
+```
+authority = "Rijkswaterstaat"
+cloud_storage.uploaded_models(authority) # to see which models have been uploaded
+
+cloud_storage.upload_model(authority, model="ijsselmeer") # to upload a new version of the `ijsselmeermodel`.
+```

notebooks/rijkswaterstaat/netwerk.py

@@ -1,13 +1,21 @@
 # %%
 import geopandas as gpd
-import pandas as pd
 from ribasim_nl import CloudStorage
-from ribasim_nl.geometry import cut_basin
+from ribasim_nl.geodataframe import direct_basins, join_by_poly_overlay, split_basins
 
 cloud = CloudStorage()
 
-# %% inlezen dataframes
+# %% Prepare RWS krw_basin_polygons
 
+krw_poly_gdf = gpd.read_file(
+    cloud.joinpath(
+        "Basisgegevens", "KRW", "krw_oppervlaktewaterlichamen_nederland_vlakken.gpkg"
+    )
+)
+
+krw_split_lines_gdf = gpd.read_file(
+    cloud.joinpath("Rijkswaterstaat", "verwerkt", "krw_split_lijnen.gpkg")
+)
 
 rws_opp_poly_gdf = gpd.read_file(
     cloud.joinpath(
@@ -16,80 +24,61 @@
 )
 
 
-# %% write it generic so it can b converted to 1 function
-krw_poly_gdf = gpd.read_file(
+rws_krw_poly_gdf = split_basins(krw_poly_gdf, krw_split_lines_gdf)
+
+rws_krw_poly_gdf = join_by_poly_overlay(
+    rws_krw_poly_gdf,
+    rws_opp_poly_gdf[["waterlichaam", "geometry"]],
+    select_by="poly_area",
+)
+
+
+rws_krw_poly = cloud.joinpath("Rijkswaterstaat", "verwerkt", "krw_basins_vlakken.gpkg")
+
+rws_krw_poly_gdf.to_file(rws_krw_poly)
+
+
+# %% create overlay with krw_lines and polygons
+krw_line_gdf = gpd.read_file(
     cloud.joinpath(
-        "Basisgegevens", "KRW", "krw_oppervlaktewaterlichamen_nederland_vlakken.gpkg"
+        "Basisgegevens", "KRW", "krw_oppervlaktewaterlichamen_nederland_lijnen.gpkg"
     )
 )
 
-krw_split_lines_gdf = gpd.read_file(
-    cloud.joinpath("Rijkswaterstaat", "verwerkt", "krw_split_lijnen.gpkg")
+rws_krw_lines = cloud.joinpath("Rijkswaterstaat", "verwerkt", "krw_basins_lijnen.gpkg")
+
+rws_krw_line_gdf = join_by_poly_overlay(
+    gpd.GeoDataFrame(krw_line_gdf.explode()["geometry"]), rws_krw_poly_gdf
 )
 
-rws_krw_basins = cloud.joinpath(
-    "Rijkswaterstaat", "verwerkt", "krw_basins_vlakken.gpkg"
+rws_krw_line_gdf.to_file(rws_krw_lines)
+
+# %% direct basins
+
+basin_ident = "owmident"
+link_ident = "Name"
+
+basins_gdf = gpd.read_file(
+    cloud.joinpath("Rijkswaterstaat", "verwerkt", "krw_basins_vlakken.gpkg")
 )
 
-lines_gdf = krw_split_lines_gdf
-poly_gdf = krw_poly_gdf
-
-for line in lines_gdf.explode(index_parts=False).itertuples():
-    # filter by spatial index
-    idx = poly_gdf.sindex.intersection(line.geometry.bounds)
-    poly_select_gdf = poly_gdf.iloc[idx][poly_gdf.iloc[idx].intersects(line.geometry)]
-
-    ## filter by intersecting geometry
-    poly_select_gdf = poly_select_gdf[poly_select_gdf.intersects(line.geometry)]
-
-    ## filter polygons with two intersection-points only
-    poly_select_gdf = poly_select_gdf[
-        poly_select_gdf.geometry.boundary.intersection(line.geometry).apply(
-            lambda x: False if x.geom_type == "Point" else len(x.geoms) == 2
-        )
-    ]
-
-    ## if there are no polygon-candidates, something is wrong
-    if poly_select_gdf.empty:
-        print(
-            f"no intersect for {line}. Please make sure it is extended outside the basin on two sides"
-        )
-    else:
-        ## we create 2 new fatures in data
-        data = []
-        for basin in poly_select_gdf.itertuples():
-            kwargs = basin._asdict()
-            for geom in cut_basin(basin.geometry, line.geometry).geoms:
-                kwargs["geometry"] = geom
-                data += [{**kwargs}]
-
-    ## we update poly_gdf with new polygons
-    poly_gdf = poly_gdf[~poly_gdf.index.isin(poly_select_gdf.index)]
-    poly_gdf = pd.concat(
-        [poly_gdf, gpd.GeoDataFrame(data, crs=poly_gdf.crs).set_index("Index")],
-        ignore_index=True,
-    )
+network_gdf = gpd.read_file(
+    cloud.joinpath("Basisgegevens", "lsm3-j18_5v6", "shapes", "network_Branches.shp")
+)
+network_gdf.set_crs(28992, inplace=True)
+drop_duplicates = True
 
-poly_gdf.to_file(rws_krw_basins)
+poly_directions_gdf = direct_basins(basins_gdf, network_gdf, basin_ident, link_ident)
 
-# %% add name
-naming_poly_gdf = rws_opp_poly_gdf
-naming_column = "waterlichaam"
 
-# start function
-columns = list(poly_gdf.columns) + [naming_column]
-poly_gdf["left_index"] = poly_gdf.index
+poly_directions_gdf.to_file(
+    cloud.joinpath("Rijkswaterstaat", "verwerkt", "krw_basins_verbindingen.gpkg")
+)
 
-overlay_gdf = gpd.overlay(poly_gdf, naming_poly_gdf, how="intersection")
-overlay_gdf["geom_area"] = overlay_gdf.geometry.area
-overlay_gdf.sort_values(by="geom_area", inplace=True)
+# %% snap nodes
 
-overlay_gdf.drop_duplicates(subset="left_index", keep="last", inplace=True)
-overlay_gdf.sort_values(by="left_index", inplace=True)
-overlay_gdf.index = poly_gdf.index
-overlay_gdf = overlay_gdf[columns]
+# %% build graph
 
-overlay_gdf.loc[:, ["geometry"]] = poly_gdf.geometry
-overlay_gdf.to_file(rws_krw_basins)
+# %% build_network
 
-# %%
+# %% A(h)

src/ribasim_nl/ribasim_nl/cloud.py

@@ -1,7 +1,10 @@
+# %%
 import logging
 import os
+import re
 import shutil
 from dataclasses import dataclass, field
+from datetime import date
 from pathlib import Path
 from xml.etree import ElementTree
 
@@ -48,6 +51,14 @@ def is_dir(item):
     return Path(item).suffix == ""
 
 
+@dataclass
+class ModelVersion:
+    model: str
+    year: int
+    month: int
+    revision: int
+
+
 @dataclass
 class CloudStorage:
     """Connect a local 'data_dir` to cloud-storage."""
@@ -327,3 +338,80 @@ def download_all(self, authority, overwrite: bool = False):
         """Download all files for authority."""
         url = self.joinurl(authority)
         self.download_content(url, overwrite=overwrite)
+
+    def uploaded_models(self, authority):
+        """Get all model versions uploaded for an authority"""
+
+        # function to strip version from a models dir
+        def strip_version(dir: str):
+            pattern = r"^(.*)_([\d]+)_([\d]+)_([\d]+)$"
+            match = re.match(pattern, dir)
+            return ModelVersion(
+                match.group(1),
+                int(match.group(2)),
+                int(match.group(3)),
+                int(match.group(4)),
+            )
+
+        # get uploaded_models
+        models_url = self.joinurl(authority, "modellen")
+        uploaded_models = self.content(models_url)
+
+        return [strip_version(i) for i in uploaded_models]
+
+    def upload_model(self, authority: str, model: str):
+        """Upload a model to a water authority
+
+        Parameters
+        ----------
+        authority : str
+            Water authority to upload a model for
+        model : str
+            name of the model (directory) to upload
+
+        Raises
+        ------
+        ValueError
+            If model does not exist locally
+        """
+
+        # get today, so we can later derive a version
+        today = date.today()
+
+        # check if model-directory exists locally
+        model_dir = self.joinpath(authority, "modellen", model)
+
+        if not model_dir.exists():
+            raise ValueError(f"""model at '{model_dir}' does not exis.""")
+
+        # check previously uploaded models to get a revision number
+        uploaded_models = self.uploaded_models(authority=authority)
+        monthly_revisions = [
+            i.revision
+            for i in uploaded_models
+            if (i.model == model)
+            and (i.year == today.year)
+            and (i.month == today.month)
+        ]
+
+        if monthly_revisions:
+            revision = max(monthly_revisions) + 1
+        else:
+            revision = 0
+
+        # create local local copy with the correct revion number
+        model_version_dir = model_dir.parent.joinpath(
+            f"{model}_{today.year}_{today.month}_{revision}"
+        )
+        model_version_dir.mkdir()
+
+        # copy model content to version dir
+        for file in model_dir.glob("*"):
+            dst_file = model_version_dir / file.name
+            dst_file.write_bytes(file.read_bytes())
+
+        # upload content to remote
+        self.create_dir(authority, "modellen", model_version_dir.name)
+        self.upload_content(model_version_dir)
+
+        return ModelVersion(model, today.year, today.month, revision)