Merge pull request #8 from ClimateImpactLab/feature/upload-nc-sliiders

Add netcdf versions of SLIIDERS and pyCIAM results to zenodo
ClimateImpactLab · Mar 4, 2024 · c53399d · c53399d
2 parents dd53baa + bda96c5
commit c53399d
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 36 deletions.
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -1,10 +1,11 @@
 History
 =======
 
-v1.1.3 (unreleased)
+v1.2 (unreleased)
 -------------------
 * Point `data-acquisition.ipynb` to updated Zenodo deposit that fixes the dtype of `subsets` variable in `diaz2016_inputs_raw.zarr.zip` to be bool rather than int8
 * Variable name bugfix in `data-acquisition.ipynb`
+* Add netcdf versions of SLIIDERS and the pyCIAM results to `upload-zenodo.ipynb`
 
 v1.1.2
 ------

diff --git a/notebooks/post-processing/zenodo-upload.ipynb b/notebooks/post-processing/zenodo-upload.ipynb
@@ -3,6 +3,17 @@
   {
    "cell_type": "code",
    "execution_count": 1,
+   "id": "42ac0087-fe1b-4088-919e-ce007e6dea8c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
    "id": "706cee66-6557-402d-ae97-679ed202a9fe",
    "metadata": {
     "tags": []
@@ -16,7 +27,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "id": "29a54dac-abdc-4a95-967f-2289fc1c9ecb",
    "metadata": {
     "tags": []
@@ -28,18 +39,33 @@
     "from pathlib import Path\n",
     "from shutil import make_archive\n",
     "from tempfile import TemporaryDirectory\n",
-    "from pathlib import Path\n",
     "\n",
     "import dask.config\n",
     "import requests\n",
     "import shared\n",
+    "import xarray as xr\n",
     "from sliiders import settings as sset\n",
+    "from sliiders.io import open_zarr\n",
     "from zarr import ZipStore"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
+   "id": "f804bd08-6edb-43f6-8bbd-7a2b98281c29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "PATH_SLIIDERS_NC = sset.PATH_SLIIDERS.parent / (sset.PATH_SLIIDERS.stem + \".nc\")\n",
+    "PATH_OUTPUTS_NC = shared.PATH_OUTPUTS.parent / (shared.PATH_OUTPUTS.stem + \".nc\")\n",
+    "\n",
+    "shared.save_dataset(shared.open_zarr(sset.PATH_SLIIDERS).load(), PATH_SLIIDERS_NC)\n",
+    "shared.save_dataset(shared.open_zarr(sset.PATH_OUTPUTS).load(), PATH_OUTPUTS_NC)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
    "id": "d37ca26f-091d-4cb6-be02-385464e0b687",
    "metadata": {
     "tags": []
@@ -48,10 +74,10 @@
     {
      "data": {
       "text/plain": [
-       "<dask.config.set at 0x112a25b70>"
+       "<dask.config.set at 0x7cfe44438890>"
       ]
      },
-     "execution_count": 3,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -70,7 +96,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 21,
    "id": "55a3e849-a554-49fa-882c-846acf76f3b8",
    "metadata": {
     "tags": [
@@ -79,19 +105,19 @@
    },
    "outputs": [],
    "source": [
-    "ACCESS_TOKEN = environ[\"ACCESS_TOKEN\"]\n",
-    "VERSION = \"1.1.0\"\n",
+    "ACCESS_TOKEN = \"Q5z5IQ1m5Z9l1QS7ZYeV78IS5bqPmhzcFVo0KSNLoh2p39HRMPgFoJsCyQt5\"\n",
+    "VERSION = \"1.2.0\"\n",
     "TITLES = {\n",
     "    \"SLIIDERS\": \"SLIIDERS: Sea Level Impacts Input Dataset by Elevation, Region, and Scenario\",\n",
-    "    \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n",
+    "    # \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n",
     "}\n",
-    "PYCIAM_CODE_PATH=Path(\"pyCIAM-1.1.2.zip\")\n",
-    "SLIIDERS_CODE_PATH=Path(\"sliiders-1.1.1.zip\")"
+    "PYCIAM_CODE_PATH = Path(\"pyCIAM-1.1.2.zip\")\n",
+    "SLIIDERS_CODE_PATH = Path(\"/tmp/sliiders-1.2.zip\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 22,
    "id": "8725b84c-8a7f-4d88-96ec-849e963ef8ce",
    "metadata": {},
    "outputs": [],
@@ -101,7 +127,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 23,
    "id": "956c40ee-e3e8-4257-bd0d-2c938546604d",
    "metadata": {},
    "outputs": [],
@@ -136,7 +162,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 24,
    "id": "b5fcb2b8-3fb4-4789-bae0-18f7a66b7c51",
    "metadata": {},
    "outputs": [],
@@ -206,15 +232,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 25,
    "id": "55188020-c02f-4fa6-bf4c-cf1cd87e1754",
    "metadata": {},
    "outputs": [],
    "source": [
     "# Files and paths\n",
     "ORIGINAL_PATHS = {\n",
     "    \"SLIIDERS\": {\n",
-    "        \"products\": [shared.PATH_SLIIDERS],\n",
+    "        \"products\": [sset.PATH_SLIIDERS, PATH_SLIIDERS_NC],\n",
     "        \"inputs\": [\n",
     "            sset.PATH_GEOG_GTSM_SNAPPED,\n",
     "            sset.PATH_GEOG_GTSM_STATIONS_TOTHIN,\n",
@@ -224,6 +250,7 @@
     "    \"pyCIAM\": {\n",
     "        \"products\": [\n",
     "            shared.PATH_OUTPUTS,\n",
+    "            PATH_OUTPUTS_NC,\n",
     "            shared.PATH_DIAZ_RES,\n",
     "            shared.PATH_MOVEFACTOR_DATA,\n",
     "        ],\n",
@@ -240,7 +267,7 @@
     "if PYCIAM_CODE_PATH is not None:\n",
     "    ORIGINAL_PATHS[\"pyCIAM\"][\"source\"] = [PYCIAM_CODE_PATH]\n",
     "if SLIIDERS_CODE_PATH is not None:\n",
-    "    ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH"
+    "    ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH]"
    ]
   },
   {
@@ -255,7 +282,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 40,
    "id": "50c10e4c-b3eb-4714-8d32-486ac878d041",
    "metadata": {},
    "outputs": [],
@@ -264,7 +291,7 @@
     "    dep = EXISTING_DEPOSITS[name]\n",
     "    # create new deposit\n",
     "    deposition_id = dep[\"id\"]\n",
-    "    if deposition_id == int(dep[\"links\"][\"latest\"].split(\"/\")[-1]):\n",
+    "    if \"latest_draft\" not in dep[\"links\"]:\n",
     "        url = f\"{Z_URL}/{deposition_id}/actions/newversion\"\n",
     "        r = requests.post(url, params=PARAMS)\n",
     "        if r.status_code not in [200, 201]:\n",
@@ -390,7 +417,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 41,
    "id": "868b1d78-5eb4-4fa6-bcfc-0dbd8c88d39b",
    "metadata": {},
    "outputs": [],
@@ -408,23 +435,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 44,
    "id": "ff8703bb-c630-418e-9f2f-c62b116ee7e8",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "pyCIAM\n",
-      "Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/raw/CIAM_2016/diaz2016_inputs_raw.zarr\n",
-      "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/slr/ar5/ar5-msl-rel-2005-quantiles.zarr\n",
-      "Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/int/exposure/ypk/finalized/ypk_2000_2100_20221122.zarr\n",
-      "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg.zarr\n",
-      "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg_adm.zarr\n",
-      "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/pyCIAM_outputs.zarr\n",
-      "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/diaz2016_outputs.zarr\n",
-      "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/suboptimal_capital_by_movefactor.zarr\n"
+      "SLIIDERS\n"
+     ]
+    },
+    {
+     "ename": "TypeError",
+     "evalue": "string indices must be integers, not 'str'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[44], line 6\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(name)\n\u001b[1;32m      4\u001b[0m this_dep \u001b[38;5;241m=\u001b[39m draft_deps[name]\n\u001b[0;32m----> 6\u001b[0m uploads[name] \u001b[38;5;241m=\u001b[39m \u001b[43mupload_file_list\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m    \u001b[49m\u001b[43mthis_dep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkind\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs/\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moverwrite\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[1;32m      8\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      9\u001b[0m uploads[name] \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m upload_file_list(\n\u001b[1;32m     10\u001b[0m     this_dep, kind[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts\u001b[39m\u001b[38;5;124m\"\u001b[39m], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts/\u001b[39m\u001b[38;5;124m\"\u001b[39m, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m     11\u001b[0m )\n\u001b[1;32m     12\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kind\u001b[38;5;241m.\u001b[39mkeys():\n",
+      "Cell \u001b[0;32mIn[40], line 83\u001b[0m, in \u001b[0;36mupload_file_list\u001b[0;34m(deposit, flist, root, overwrite)\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m     82\u001b[0m     out \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 83\u001b[0m     existing_files \u001b[38;5;241m=\u001b[39m \u001b[43m{\u001b[49m\n\u001b[1;32m     84\u001b[0m \u001b[43m        \u001b[49m\u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\n\u001b[1;32m     85\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdeposit\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlinks\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfiles\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mPARAMS\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     86\u001b[0m \u001b[43m    \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m     87\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m     88\u001b[0m         \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
+      "Cell \u001b[0;32mIn[40], line 84\u001b[0m, in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m     82\u001b[0m     out \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     83\u001b[0m     existing_files \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m---> 84\u001b[0m         \u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m: f\n\u001b[1;32m     85\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mget(deposit[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfiles\u001b[39m\u001b[38;5;124m\"\u001b[39m], params\u001b[38;5;241m=\u001b[39mPARAMS)\u001b[38;5;241m.\u001b[39mjson()\n\u001b[1;32m     86\u001b[0m     }\n\u001b[1;32m     87\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m     88\u001b[0m         \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
+      "\u001b[0;31mTypeError\u001b[0m: string indices must be integers, not 'str'"
      ]
     }
    ],
@@ -440,7 +472,7 @@
     "    uploads[name] += upload_file_list(\n",
     "        this_dep, kind[\"products\"], \"products/\", overwrite=False\n",
     "    )\n",
-    "    if \"source\" in kind.keys(): \n",
+    "    if \"source\" in kind.keys():\n",
     "        uploads[name] += upload_file_list(\n",
     "            this_dep, kind[\"source\"], \"source/\", overwrite=False\n",
     "        )"
@@ -449,9 +481,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python [conda env:sliiders]",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "conda-env-sliiders-py"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -463,7 +495,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.9"
+   "version": "3.11.8"
   },
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {

diff --git a/notebooks/shared.py b/notebooks/shared.py
@@ -9,7 +9,7 @@
 
 DIR_SCRATCH = Path("/tmp/ciam-scratch")
 
-SLIIDERS_VERS = "v1.1"
+SLIIDERS_VERS = "v1.2"
 RES_VERS = "v1.1"
 
 # Cloud Storage tools (will work with local storage as well but may need to be specifiec
@@ -223,6 +223,12 @@ def open_dataset(path, **kwargs):
     return xr.open_dataset(_path, **kwargs)
 
 
+def save_dataset(ds, path, **kwargs):
+    _path = str(_to_fuse(path))
+    _generate_parent_fuse_dirs(_path)
+    return ds.to_netcdf(_path, **kwargs)
+
+
 def open_dataarray(path, **kwargs):
     _path = str(_to_fuse(path))
     _generate_parent_fuse_dirs(_path)