diff --git a/HISTORY.rst b/HISTORY.rst index f85c4e4..23d138e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,10 +1,11 @@ History ======= -v1.1.3 (unreleased) +v1.2 (unreleased) ------------------- * Point `data-acquisition.ipynb` to updated Zenodo deposit that fixes the dtype of `subsets` variable in `diaz2016_inputs_raw.zarr.zip` to be bool rather than int8 * Variable name bugfix in `data-acquisition.ipynb` +* Add netcdf versions of SLIIDERS and the pyCIAM results to `upload-zenodo.ipynb` v1.1.2 ------ diff --git a/notebooks/post-processing/zenodo-upload.ipynb b/notebooks/post-processing/zenodo-upload.ipynb index aaea54c..8394580 100644 --- a/notebooks/post-processing/zenodo-upload.ipynb +++ b/notebooks/post-processing/zenodo-upload.ipynb @@ -3,6 +3,17 @@ { "cell_type": "code", "execution_count": 1, + "id": "42ac0087-fe1b-4088-919e-ce007e6dea8c", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "id": "706cee66-6557-402d-ae97-679ed202a9fe", "metadata": { "tags": [] @@ -16,7 +27,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "29a54dac-abdc-4a95-967f-2289fc1c9ecb", "metadata": { "tags": [] @@ -28,18 +39,33 @@ "from pathlib import Path\n", "from shutil import make_archive\n", "from tempfile import TemporaryDirectory\n", - "from pathlib import Path\n", "\n", "import dask.config\n", "import requests\n", "import shared\n", + "import xarray as xr\n", "from sliiders import settings as sset\n", + "from sliiders.io import open_zarr\n", "from zarr import ZipStore" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, + "id": "f804bd08-6edb-43f6-8bbd-7a2b98281c29", + "metadata": {}, + "outputs": [], + "source": [ + "PATH_SLIIDERS_NC = sset.PATH_SLIIDERS.parent / (sset.PATH_SLIIDERS.stem + \".nc\")\n", + "PATH_OUTPUTS_NC = shared.PATH_OUTPUTS.parent / (shared.PATH_OUTPUTS.stem + \".nc\")\n", + "\n", + "shared.save_dataset(shared.open_zarr(sset.PATH_SLIIDERS).load(), PATH_SLIIDERS_NC)\n", + "shared.save_dataset(shared.open_zarr(sset.PATH_OUTPUTS).load(), PATH_OUTPUTS_NC)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "id": "d37ca26f-091d-4cb6-be02-385464e0b687", "metadata": { "tags": [] @@ -48,10 +74,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -70,7 +96,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 21, "id": "55a3e849-a554-49fa-882c-846acf76f3b8", "metadata": { "tags": [ @@ -79,19 +105,19 @@ }, "outputs": [], "source": [ - "ACCESS_TOKEN = environ[\"ACCESS_TOKEN\"]\n", - "VERSION = \"1.1.0\"\n", + "ACCESS_TOKEN = \"Q5z5IQ1m5Z9l1QS7ZYeV78IS5bqPmhzcFVo0KSNLoh2p39HRMPgFoJsCyQt5\"\n", + "VERSION = \"1.2.0\"\n", "TITLES = {\n", " \"SLIIDERS\": \"SLIIDERS: Sea Level Impacts Input Dataset by Elevation, Region, and Scenario\",\n", - " \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n", + " # \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n", "}\n", - "PYCIAM_CODE_PATH=Path(\"pyCIAM-1.1.2.zip\")\n", - "SLIIDERS_CODE_PATH=Path(\"sliiders-1.1.1.zip\")" + "PYCIAM_CODE_PATH = Path(\"pyCIAM-1.1.2.zip\")\n", + "SLIIDERS_CODE_PATH = Path(\"/tmp/sliiders-1.2.zip\")" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 22, "id": "8725b84c-8a7f-4d88-96ec-849e963ef8ce", "metadata": {}, "outputs": [], @@ -101,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 23, "id": "956c40ee-e3e8-4257-bd0d-2c938546604d", "metadata": {}, "outputs": [], @@ -136,7 +162,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 24, "id": "b5fcb2b8-3fb4-4789-bae0-18f7a66b7c51", "metadata": {}, "outputs": [], @@ -206,7 +232,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 25, "id": "55188020-c02f-4fa6-bf4c-cf1cd87e1754", "metadata": {}, "outputs": [], @@ -214,7 +240,7 @@ "# Files and paths\n", "ORIGINAL_PATHS = {\n", " \"SLIIDERS\": {\n", - " \"products\": [shared.PATH_SLIIDERS],\n", + " \"products\": [sset.PATH_SLIIDERS, PATH_SLIIDERS_NC],\n", " \"inputs\": [\n", " sset.PATH_GEOG_GTSM_SNAPPED,\n", " sset.PATH_GEOG_GTSM_STATIONS_TOTHIN,\n", @@ -224,6 +250,7 @@ " \"pyCIAM\": {\n", " \"products\": [\n", " shared.PATH_OUTPUTS,\n", + " PATH_OUTPUTS_NC,\n", " shared.PATH_DIAZ_RES,\n", " shared.PATH_MOVEFACTOR_DATA,\n", " ],\n", @@ -240,7 +267,7 @@ "if PYCIAM_CODE_PATH is not None:\n", " ORIGINAL_PATHS[\"pyCIAM\"][\"source\"] = [PYCIAM_CODE_PATH]\n", "if SLIIDERS_CODE_PATH is not None:\n", - " ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH" + " ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH]" ] }, { @@ -255,7 +282,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 40, "id": "50c10e4c-b3eb-4714-8d32-486ac878d041", "metadata": {}, "outputs": [], @@ -264,7 +291,7 @@ " dep = EXISTING_DEPOSITS[name]\n", " # create new deposit\n", " deposition_id = dep[\"id\"]\n", - " if deposition_id == int(dep[\"links\"][\"latest\"].split(\"/\")[-1]):\n", + " if \"latest_draft\" not in dep[\"links\"]:\n", " url = f\"{Z_URL}/{deposition_id}/actions/newversion\"\n", " r = requests.post(url, params=PARAMS)\n", " if r.status_code not in [200, 201]:\n", @@ -390,7 +417,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 41, "id": "868b1d78-5eb4-4fa6-bcfc-0dbd8c88d39b", "metadata": {}, "outputs": [], @@ -408,7 +435,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 44, "id": "ff8703bb-c630-418e-9f2f-c62b116ee7e8", "metadata": {}, "outputs": [ @@ -416,15 +443,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "pyCIAM\n", - "Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/raw/CIAM_2016/diaz2016_inputs_raw.zarr\n", - "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/slr/ar5/ar5-msl-rel-2005-quantiles.zarr\n", - "Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/int/exposure/ypk/finalized/ypk_2000_2100_20221122.zarr\n", - "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg.zarr\n", - "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg_adm.zarr\n", - "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/pyCIAM_outputs.zarr\n", - "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/diaz2016_outputs.zarr\n", - "Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/suboptimal_capital_by_movefactor.zarr\n" + "SLIIDERS\n" + ] + }, + { + "ename": "TypeError", + "evalue": "string indices must be integers, not 'str'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[44], line 6\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(name)\n\u001b[1;32m 4\u001b[0m this_dep \u001b[38;5;241m=\u001b[39m draft_deps[name]\n\u001b[0;32m----> 6\u001b[0m uploads[name] \u001b[38;5;241m=\u001b[39m \u001b[43mupload_file_list\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43mthis_dep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkind\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs/\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moverwrite\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[1;32m 8\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 9\u001b[0m uploads[name] \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m upload_file_list(\n\u001b[1;32m 10\u001b[0m this_dep, kind[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts\u001b[39m\u001b[38;5;124m\"\u001b[39m], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts/\u001b[39m\u001b[38;5;124m\"\u001b[39m, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 11\u001b[0m )\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kind\u001b[38;5;241m.\u001b[39mkeys():\n", + "Cell \u001b[0;32mIn[40], line 83\u001b[0m, in \u001b[0;36mupload_file_list\u001b[0;34m(deposit, flist, root, overwrite)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 82\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 83\u001b[0m existing_files \u001b[38;5;241m=\u001b[39m \u001b[43m{\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\n\u001b[1;32m 85\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdeposit\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlinks\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfiles\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mPARAMS\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 86\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", + "Cell \u001b[0;32mIn[40], line 84\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 82\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 83\u001b[0m existing_files \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m---> 84\u001b[0m \u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m: f\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mget(deposit[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfiles\u001b[39m\u001b[38;5;124m\"\u001b[39m], params\u001b[38;5;241m=\u001b[39mPARAMS)\u001b[38;5;241m.\u001b[39mjson()\n\u001b[1;32m 86\u001b[0m }\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mTypeError\u001b[0m: string indices must be integers, not 'str'" ] } ], @@ -440,7 +472,7 @@ " uploads[name] += upload_file_list(\n", " this_dep, kind[\"products\"], \"products/\", overwrite=False\n", " )\n", - " if \"source\" in kind.keys(): \n", + " if \"source\" in kind.keys():\n", " uploads[name] += upload_file_list(\n", " this_dep, kind[\"source\"], \"source/\", overwrite=False\n", " )" @@ -449,9 +481,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:sliiders]", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-sliiders-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -463,7 +495,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.11.8" }, "widgets": { "application/vnd.jupyter.widget-state+json": { diff --git a/notebooks/shared.py b/notebooks/shared.py index e8fb5e5..0825bb2 100644 --- a/notebooks/shared.py +++ b/notebooks/shared.py @@ -9,7 +9,7 @@ DIR_SCRATCH = Path("/tmp/ciam-scratch") -SLIIDERS_VERS = "v1.1" +SLIIDERS_VERS = "v1.2" RES_VERS = "v1.1" # Cloud Storage tools (will work with local storage as well but may need to be specifiec @@ -223,6 +223,12 @@ def open_dataset(path, **kwargs): return xr.open_dataset(_path, **kwargs) +def save_dataset(ds, path, **kwargs): + _path = str(_to_fuse(path)) + _generate_parent_fuse_dirs(_path) + return ds.to_netcdf(_path, **kwargs) + + def open_dataarray(path, **kwargs): _path = str(_to_fuse(path)) _generate_parent_fuse_dirs(_path)