Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update to SLIIDERS v1.2 #9

Merged
merged 2 commits into from
Apr 3, 2024
Merged

update to SLIIDERS v1.2 #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4c50b79
update to SLIIDERS v1.2
bolliger32 Apr 3, 2024
b07a279
autoformat
bolliger32 Apr 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions HISTORY.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
History
=======

v1.2 (unreleased)
-------------------
v1.2.0
------
* Point `data-acquisition.ipynb` to updated Zenodo deposit that fixes the dtype of `subsets` variable in `diaz2016_inputs_raw.zarr.zip` to be bool rather than int8
* Variable name bugfix in `data-acquisition.ipynb`
* Add netcdf versions of SLIIDERS and the pyCIAM results to `upload-zenodo.ipynb`
* Update results in Zenodo record to use SLIIDERS v1.2

v1.1.2
------
Expand Down
136 changes: 64 additions & 72 deletions notebooks/models/create-surge-lookup-tables.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,23 @@
{
"cell_type": "code",
"execution_count": 1,
"id": "6b183aa3-366a-4f5f-bb95-5ef4bf48d438",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "cd4b2928-65a0-4733-9c71-fbefa85590be",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import sys\n",
Expand All @@ -41,16 +56,26 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 3,
"id": "afd03880-8e46-4d81-867d-bfb6505ea788",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/srv/conda/envs/notebook/lib/python3.9/site-packages/dask_gateway/client.py:21: FutureWarning: format_bytes is deprecated and will be removed in a future release. Please use dask.utils.format_bytes instead.\n",
" from distributed.utils import LoopRunner, format_bytes\n"
"/srv/conda/envs/notebook/lib/python3.10/site-packages/google/cloud/storage/transfer_manager.py:30: UserWarning: The module `transfer_manager` is a preview feature. Functionality and API may change. This warning will be removed in a future release.\n",
" warnings.warn(\n",
"/home/jovyan/git-repos/pyCIAM-public/notebooks/models/../shared.py:3: UserWarning: Shapely 2.0 is installed, but because PyGEOS is also installed, GeoPandas will still use PyGEOS by default for now. To force to use and test Shapely 2.0, you have to set the environment variable USE_PYGEOS=0. You can do this before starting the Python process, or in your code before importing geopandas:\n",
"\n",
"import os\n",
"os.environ['USE_PYGEOS'] = '0'\n",
"import geopandas\n",
"\n",
"In a future release, GeoPandas will switch to using Shapely by default. If you are using PyGEOS directly (calling PyGEOS functions on geometries from GeoPandas), this will then stop working and you are encouraged to migrate from PyGEOS to Shapely 2.0 (https://shapely.readthedocs.io/en/latest/migration_pygeos.html).\n",
" import geopandas as gpd\n"
]
}
],
Expand All @@ -75,7 +100,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"id": "47979c48-7fff-49b4-b445-ea6bf746fa37",
"metadata": {
"tags": [
Expand All @@ -86,18 +111,20 @@
"source": [
"# When running on larger/scalable dask cluster, may wish to specify number of workers\n",
"# Default is LocalCluster which will use the number of CPUs available on local machine\n",
"# N_WORKERS_MIN = 7\n",
"# N_WORKERS_MAX = 700\n",
"N_WORKERS_MIN = 7\n",
"N_WORKERS_MAX = 700\n",
"SEG_CHUNKSIZE = 5\n",
"\n",
"PARAMS = pd.read_json(PATH_PARAMS)[\"values\"]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 5,
"id": "6104bced",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"DMF_I = getattr(damage_funcs, PARAMS.dmf + \"_i\")\n",
Expand All @@ -106,70 +133,31 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"id": "dbb04b4a-14b8-4403-ad33-88bfe71bd8fc",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
" <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-f992a2c6-c6d1-11ed-862e-665816eadc72</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
"\n",
" <tr>\n",
" \n",
" <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
" <td style=\"text-align: left;\"><strong>Cluster type:</strong> dask_gateway.GatewayCluster</td>\n",
" \n",
" </tr>\n",
"\n",
" \n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status\" target=\"_blank\">/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" \n",
"\n",
" </table>\n",
"\n",
" \n",
" <details>\n",
" <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
" <div style='background-color: #f2f2f2; display: inline-block; padding: 10px; border: 1px solid #999999;'>\n",
" <h3>GatewayCluster</h3>\n",
" <ul>\n",
" <li><b>Name: </b>daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454\n",
" <li><b>Dashboard: </b><a href='/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status' target='_blank'>/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status</a>\n",
" </ul>\n",
"</div>\n",
"\n",
" </details>\n",
" \n",
"\n",
" </div>\n",
"</div>"
],
"application/vnd.jupyter.widget-view+json": {
"model_id": "fcadb94377b4430d9c2964ac9c808c2a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"<Client: 'tls://10.4.102.2:8786' processes=0 threads=0, memory=0 B>"
"VBox(children=(HTML(value='<h2>GatewayCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n<style scoped>\\n …"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
"output_type": "display_data"
}
],
"source": [
"client = start_dask_cluster(\n",
" # n_workers_min=N_WORKERS_MIN,\n",
" # n_workers_max=N_WORKERS_MAX,\n",
")\n",
"client"
"client, cluster = start_dask_cluster()\n",
"cluster.adapt(minimum=N_WORKERS_MIN, maximum=N_WORKERS_MAX)\n",
"cluster"
]
},
{
Expand All @@ -182,7 +170,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 7,
"id": "1e823775-0d12-4e52-93d1-7e3586913e65",
"metadata": {
"tags": []
Expand All @@ -205,7 +193,7 @@
" start_year=PARAMS.model_start,\n",
" slr_0_years=PARAMS.slr_0_year,\n",
" client=client,\n",
" # client_kwargs={\"batch_size\": N_WORKERS_MAX},\n",
" client_kwargs={\"batch_size\": N_WORKERS_MAX},\n",
" force_overwrite=True,\n",
" seg_chunksize=SEG_CHUNKSIZE,\n",
" mc_dim=\"quantile\",\n",
Expand All @@ -223,9 +211,11 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 11,
"id": "f941a509-7035-4105-8b79-88a97e759737",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# ensure completion and close cluster\n",
Expand All @@ -237,23 +227,25 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 12,
"id": "15563792-1ba0-435a-9981-e2b15eaa8dcd",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"(None, None)"
]
},
"execution_count": 10,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.cluster.close(), client.close()"
"cluster.close(), client.close()"
]
}
],
Expand All @@ -273,7 +265,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.10"
"version": "3.10.10"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
Expand Down
Loading
Loading