Skip to content

Commit

Permalink
add netcdfs to upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@bolliger32
bolliger32 committed Mar 4, 2024
1 parent dd53baa commit afdf3f7
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 35 deletions.
100 changes: 66 additions & 34 deletions notebooks/post-processing/zenodo-upload.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,17 @@
{
"cell_type": "code",
"execution_count": 1,
"id": "42ac0087-fe1b-4088-919e-ce007e6dea8c",
"metadata": {},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "706cee66-6557-402d-ae97-679ed202a9fe",
"metadata": {
"tags": []
Expand All @@ -16,7 +27,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 3,
"id": "29a54dac-abdc-4a95-967f-2289fc1c9ecb",
"metadata": {
"tags": []
Expand All @@ -28,18 +39,33 @@
"from pathlib import Path\n",
"from shutil import make_archive\n",
"from tempfile import TemporaryDirectory\n",
"from pathlib import Path\n",
"\n",
"import dask.config\n",
"import requests\n",
"import shared\n",
"import xarray as xr\n",
"from sliiders import settings as sset\n",
"from sliiders.io import open_zarr\n",
"from zarr import ZipStore"
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"id": "f804bd08-6edb-43f6-8bbd-7a2b98281c29",
"metadata": {},
"outputs": [],
"source": [
"PATH_SLIIDERS_NC = sset.PATH_SLIIDERS.parent / (sset.PATH_SLIIDERS.stem + \".nc\")\n",
"PATH_OUTPUTS_NC = shared.PATH_OUTPUTS.parent / (shared.PATH_OUTPUTS.stem + \".nc\")\n",
"\n",
"shared.save_dataset(shared.open_zarr(sset.PATH_SLIIDERS).load(), PATH_SLIIDERS_NC)\n",
"shared.save_dataset(shared.open_zarr(sset.PATH_OUTPUTS).load(), PATH_OUTPUTS_NC)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d37ca26f-091d-4cb6-be02-385464e0b687",
"metadata": {
"tags": []
Expand All @@ -48,10 +74,10 @@
{
"data": {
"text/plain": [
"<dask.config.set at 0x112a25b70>"
"<dask.config.set at 0x7cfe44438890>"
]
},
"execution_count": 3,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -70,7 +96,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 21,
"id": "55a3e849-a554-49fa-882c-846acf76f3b8",
"metadata": {
"tags": [
Expand All @@ -79,19 +105,19 @@
},
"outputs": [],
"source": [
"ACCESS_TOKEN = environ[\"ACCESS_TOKEN\"]\n",
"VERSION = \"1.1.0\"\n",
"ACCESS_TOKEN = \"Q5z5IQ1m5Z9l1QS7ZYeV78IS5bqPmhzcFVo0KSNLoh2p39HRMPgFoJsCyQt5\"\n",
"VERSION = \"1.2.0\"\n",
"TITLES = {\n",
" \"SLIIDERS\": \"SLIIDERS: Sea Level Impacts Input Dataset by Elevation, Region, and Scenario\",\n",
" \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n",
" # \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n",
"}\n",
"PYCIAM_CODE_PATH=Path(\"pyCIAM-1.1.2.zip\")\n",
"SLIIDERS_CODE_PATH=Path(\"sliiders-1.1.1.zip\")"
"PYCIAM_CODE_PATH = Path(\"pyCIAM-1.1.2.zip\")\n",
"SLIIDERS_CODE_PATH = Path(\"/tmp/sliiders-1.2.zip\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 22,
"id": "8725b84c-8a7f-4d88-96ec-849e963ef8ce",
"metadata": {},
"outputs": [],
Expand All @@ -101,7 +127,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 23,
"id": "956c40ee-e3e8-4257-bd0d-2c938546604d",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -136,7 +162,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 24,
"id": "b5fcb2b8-3fb4-4789-bae0-18f7a66b7c51",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -206,15 +232,15 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 25,
"id": "55188020-c02f-4fa6-bf4c-cf1cd87e1754",
"metadata": {},
"outputs": [],
"source": [
"# Files and paths\n",
"ORIGINAL_PATHS = {\n",
" \"SLIIDERS\": {\n",
" \"products\": [shared.PATH_SLIIDERS],\n",
" \"products\": [sset.PATH_SLIIDERS, PATH_SLIIDERS_NC],\n",
" \"inputs\": [\n",
" sset.PATH_GEOG_GTSM_SNAPPED,\n",
" sset.PATH_GEOG_GTSM_STATIONS_TOTHIN,\n",
Expand All @@ -224,6 +250,7 @@
" \"pyCIAM\": {\n",
" \"products\": [\n",
" shared.PATH_OUTPUTS,\n",
" PATH_OUTPUTS_NC,\n",
" shared.PATH_DIAZ_RES,\n",
" shared.PATH_MOVEFACTOR_DATA,\n",
" ],\n",
Expand All @@ -240,7 +267,7 @@
"if PYCIAM_CODE_PATH is not None:\n",
" ORIGINAL_PATHS[\"pyCIAM\"][\"source\"] = [PYCIAM_CODE_PATH]\n",
"if SLIIDERS_CODE_PATH is not None:\n",
" ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH"
" ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH]"
]
},
{
Expand All @@ -255,7 +282,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 40,
"id": "50c10e4c-b3eb-4714-8d32-486ac878d041",
"metadata": {},
"outputs": [],
Expand All @@ -264,7 +291,7 @@
" dep = EXISTING_DEPOSITS[name]\n",
" # create new deposit\n",
" deposition_id = dep[\"id\"]\n",
" if deposition_id == int(dep[\"links\"][\"latest\"].split(\"/\")[-1]):\n",
" if \"latest_draft\" not in dep[\"links\"]:\n",
" url = f\"{Z_URL}/{deposition_id}/actions/newversion\"\n",
" r = requests.post(url, params=PARAMS)\n",
" if r.status_code not in [200, 201]:\n",
Expand Down Expand Up @@ -390,7 +417,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 41,
"id": "868b1d78-5eb4-4fa6-bcfc-0dbd8c88d39b",
"metadata": {},
"outputs": [],
Expand All @@ -408,23 +435,28 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 44,
"id": "ff8703bb-c630-418e-9f2f-c62b116ee7e8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pyCIAM\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/raw/CIAM_2016/diaz2016_inputs_raw.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/slr/ar5/ar5-msl-rel-2005-quantiles.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/int/exposure/ypk/finalized/ypk_2000_2100_20221122.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg_adm.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/pyCIAM_outputs.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/diaz2016_outputs.zarr\n",
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/suboptimal_capital_by_movefactor.zarr\n"
"SLIIDERS\n"
]
},
{
"ename": "TypeError",
"evalue": "string indices must be integers, not 'str'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[44], line 6\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(name)\n\u001b[1;32m 4\u001b[0m this_dep \u001b[38;5;241m=\u001b[39m draft_deps[name]\n\u001b[0;32m----> 6\u001b[0m uploads[name] \u001b[38;5;241m=\u001b[39m \u001b[43mupload_file_list\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43mthis_dep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkind\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs/\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moverwrite\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[1;32m 8\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 9\u001b[0m uploads[name] \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m upload_file_list(\n\u001b[1;32m 10\u001b[0m this_dep, kind[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts\u001b[39m\u001b[38;5;124m\"\u001b[39m], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts/\u001b[39m\u001b[38;5;124m\"\u001b[39m, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 11\u001b[0m )\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kind\u001b[38;5;241m.\u001b[39mkeys():\n",
"Cell \u001b[0;32mIn[40], line 83\u001b[0m, in \u001b[0;36mupload_file_list\u001b[0;34m(deposit, flist, root, overwrite)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 82\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 83\u001b[0m existing_files \u001b[38;5;241m=\u001b[39m \u001b[43m{\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\n\u001b[1;32m 85\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdeposit\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlinks\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfiles\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mPARAMS\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 86\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
"Cell \u001b[0;32mIn[40], line 84\u001b[0m, in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 82\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 83\u001b[0m existing_files \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m---> 84\u001b[0m \u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m: f\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mget(deposit[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfiles\u001b[39m\u001b[38;5;124m\"\u001b[39m], params\u001b[38;5;241m=\u001b[39mPARAMS)\u001b[38;5;241m.\u001b[39mjson()\n\u001b[1;32m 86\u001b[0m }\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
"\u001b[0;31mTypeError\u001b[0m: string indices must be integers, not 'str'"
]
}
],
Expand All @@ -440,7 +472,7 @@
" uploads[name] += upload_file_list(\n",
" this_dep, kind[\"products\"], \"products/\", overwrite=False\n",
" )\n",
" if \"source\" in kind.keys(): \n",
" if \"source\" in kind.keys():\n",
" uploads[name] += upload_file_list(\n",
" this_dep, kind[\"source\"], \"source/\", overwrite=False\n",
" )"
Expand All @@ -449,9 +481,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:sliiders]",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "conda-env-sliiders-py"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -463,7 +495,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
"version": "3.11.8"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
Expand Down
8 changes: 7 additions & 1 deletion notebooks/shared.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

DIR_SCRATCH = Path("/tmp/ciam-scratch")

SLIIDERS_VERS = "v1.1"
SLIIDERS_VERS = "v1.2"
RES_VERS = "v1.1"

# Cloud Storage tools (will work with local storage as well but may need to be specifiec
Expand Down Expand Up @@ -223,6 +223,12 @@ def open_dataset(path, **kwargs):
return xr.open_dataset(_path, **kwargs)


def save_dataset(ds, path, **kwargs):
_path = str(_to_fuse(path))
_generate_parent_fuse_dirs(_path)
return ds.to_netcdf(_path, **kwargs)


def open_dataarray(path, **kwargs):
_path = str(_to_fuse(path))
_generate_parent_fuse_dirs(_path)
Expand Down

0 comments on commit afdf3f7

Please sign in to comment.