add missing notebooks

notebooks/dommel/get_data.py

@@ -0,0 +1,9 @@
+# %%
+from ribasim_nl import CloudStorage
+
+cloud = CloudStorage()
+
+dommel_url = cloud.joinurl("dommel", "verwerkt")
+
+# %%
+cloud.download_content(dommel_url)

notebooks/vrij_afwaterend/hydamo_0_analyse_data_waterboard.ipynb

@@ -0,0 +1,298 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "import pandas as pd\n",
+    "from pandas_xlsx_tables import xlsx_tables_to_dfs\n",
+    "\n",
+    "sys.path.append(\"Ribasim-NL\\\\src\\\\hydamo\")\n",
+    "from hydamo.datamodel import HyDAMO\n",
+    "from ribasim_lumping_tools.LHM_data_bewerking_analyse_utils import (\n",
+    "    check_ids_hydamo_data,\n",
+    "    check_if_object_on_hydroobject,\n",
+    "    read_original_data,\n",
+    "    translate_data_to_hydamo_format,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2",
+   "metadata": {},
+   "source": [
+    "Vertaal originele data naar Hydamo data zoals gedefinieerd in de tabel hydamo_data_format.xlsx"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "base_dir = \"..\\\\\"\n",
+    "\n",
+    "waterboard = \"AAenMaas\"\n",
+    "waterboard_code = 1\n",
+    "path_hydamo_format = None  # Daniel Tollenaar: toegevoegd opm 14/8 i.v.m. pre-commit\n",
+    "hydamo_format = None  # Daniel Tollenaar: toegevoegd opm 14/8 i.v.m. pre-commit"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "waterboard_dir = Path(base_dir, waterboard, \"verwerkt\")\n",
+    "path_hydamo_format_table = Path(waterboard_dir, \"HyDAMO_format_AAenMaas.xlsx\")\n",
+    "hydamo_format_table = xlsx_tables_to_dfs(path_hydamo_format)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# eerst inlezen hydroobject, vertalen naar hydamo\n",
+    "hydamo_object = \"hydroobject\"\n",
+    "hydamo_translate_table, data_original = read_original_data(waterboard_dir, hydamo_format, hydamo_object, waterboard)\n",
+    "hydroobject = translate_data_to_hydamo_format(hydamo_translate_table, data_original)\n",
+    "\n",
+    "# maak een created_date aan indien nodig\n",
+    "if \"created_date\" not in data_original.columns:\n",
+    "    hydroobject[\"created_date\"] = pd.NaT\n",
+    "# transformeer created_date waardes indien nodig\n",
+    "hydroobject[\"created_date\"] = hydroobject[\"created_date\"].replace(\"\", pd.NaT)\n",
+    "\n",
+    "# hydroobject.loc[hydroobject['code'].duplicated(keep=False), 'data_issue'] = 'duplicate_id'\n",
+    "data_hydamo_dict = dict(hydroobject=hydroobject.set_crs(28992))  # NOQA (FIXME: voor passen precommit)\n",
+    "\n",
+    "# geometry hydroobject bufferen met 10 cm voor de spatial join\n",
+    "hydroobject[\"buffer\"] = hydroobject.copy().buffer(5)  # 5 meter buffer omdat anders relevante gemalen wegvallen\n",
+    "hydroobject_buffered = hydroobject.set_geometry(\"buffer\").set_crs(28992)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6",
+   "metadata": {},
+   "source": [
+    "Specificeer welke HyDAMO data je wilt omzetten"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "hydamo_objects = [\n",
+    "    \"stuw\",\n",
+    "    \"gemaal\",\n",
+    "    \"afvoergebiedaanvoergebied\",\n",
+    "    \"pomp\",\n",
+    "    ##'peilgebiedvigerend',\n",
+    "    ##'peilgebiedpraktijk',\n",
+    "    ##'streefpeil',\n",
+    "    \"duikersifonhevel\",\n",
+    "    ##'afsluiter',\n",
+    "    ##'sluis',\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "for hydamo_object in hydamo_objects:\n",
+    "    # lees aangeleverde data en hydamo tabel voor gegeven kunstwerk en waterschap\n",
+    "    table_hydamo, data_original = read_original_data(waterboard_dir, hydamo_format, hydamo_object, waterboard)\n",
+    "    if data_original is None:\n",
+    "        data_hydamo_dict[hydamo_object] = None\n",
+    "    else:\n",
+    "        # vertaal data naar hydamo-ribasim format\n",
+    "        data_hydamo = translate_data_to_hydamo_format(table_hydamo, data_original)\n",
+    "\n",
+    "        # maak een created_date aan indien nodig\n",
+    "        if \"created_date\" not in data_original.columns and hydamo_object != \"sluis\":\n",
+    "            hydroobject[\"created_date\"] = pd.NaT\n",
+    "        if \"last_edited_date\" not in data_original.columns and hydamo_object == \"afsluiter\":\n",
+    "            hydroobject[\"last_edited_date\"] = pd.NaT\n",
+    "        if \"lvpublicatiedatum\" not in data_original.columns and hydamo_object == \"afsluiter\":\n",
+    "            hydroobject[\"lvpublicatiedatum\"] = pd.NaT\n",
+    "\n",
+    "        # transformeer created_date waardes indien nodig\n",
+    "        if hydamo_object != \"sluis\":\n",
+    "            data_hydamo[\"created_date\"] = data_hydamo[\"created_date\"].replace(\"\", pd.NaT)\n",
+    "        if hydamo_object == \"afsluiter\":\n",
+    "            data_hydamo[\"last_edited_date\"] = data_hydamo[\"last_edited_date\"].replace(\"\", pd.NaT)\n",
+    "            data_hydamo[\"lvpublicatiedatum\"] = data_hydamo[\"lvpublicatiedatum\"].replace(\"\", pd.NaT)\n",
+    "\n",
+    "        # check dubbele id's\n",
+    "        if hydamo_object not in [\"streefpeil\"]:  # streefpeil heeft geen code, alleen globalid etc\n",
+    "            data_hydamo.loc[data_hydamo[\"code\"].duplicated(keep=False), \"data_issue\"] = \"duplicate_id\"\n",
+    "            # TODO check op 'code' lijkt met logischer want die kolom wordt vaker gebruikt. Maar bij WDOD bijv. is die niet ingevuld. Toch op globalid?\n",
+    "        # check of kuntstwerk op hydroobject ligt\n",
+    "        if hydamo_object in [\"stuw\", \"gemaal\", \"duikersifonhevel\", \"sluis\"]:\n",
+    "            data_hydamo = check_if_object_on_hydroobject(\n",
+    "                data_hydamo=data_hydamo, hydroobject_buffered=hydroobject_buffered\n",
+    "            )\n",
+    "            # verwijder kunstwerken die niet op hydroobject liggen\n",
+    "            data_hydamo = data_hydamo[data_hydamo[\"code_hydroobject\"] != \"niet op hydroobject\"]\n",
+    "            data_hydamo = data_hydamo.reset_index()\n",
+    "        # voeg toe aan de hydamo dataset\n",
+    "        data_hydamo_dict[hydamo_object] = data_hydamo"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9",
+   "metadata": {},
+   "source": [
+    "Waterschap specifieke acties"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "10",
+   "metadata": {},
+   "source": [
+    "Export normal"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# for hydamo_object in ['hydroobject'] + hydamo_objects:\n",
+    "#     # export to geopackage\n",
+    "#     export_to_geopackage(\n",
+    "#         data_hydamo=data_hydamo_dict[hydamo_object],\n",
+    "#         hydamo_format=hydamo_format,\n",
+    "#         waterboard=waterboard,\n",
+    "#         hydamo_object=hydamo_object\n",
+    "#     )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "12",
+   "metadata": {},
+   "source": [
+    "### ribasim-nl hydamo"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "hydamo = HyDAMO(version=\"2.2.1_sweco\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for hydamo_object in [\"hydroobject\"] + hydamo_objects:\n",
+    "    data_hydamo = data_hydamo_dict[hydamo_object]\n",
+    "    if hydamo_object == \"stuw\":\n",
+    "        data_hydamo = data_hydamo.drop(columns=[\"code_hydroobject\", \"data_issue\"])  # ,'index_right'\n",
+    "    data_hydamo = check_ids_hydamo_data(data_hydamo, waterboard_code, hydamo_object)\n",
+    "    setattr(hydamo, hydamo_object, data_hydamo)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "hydamo.to_geopackage(\"..\\\\hydamo.gpkg\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "ribasim_lumping_venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}