Add PAVICS homepage notebooks to Jenkins testsuite

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb

@@ -43,7 +43,7 @@
    ],
    "source": [
     "from siphon.catalog import TDSCatalog\n",
-    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n",
+    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"  # TEST_USE_PROD_DATA\n",
     "\n",
     "# Create Catalog\n",
     "cat = TDSCatalog(url)\n",
@@ -777,6 +777,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "import xarray as xr\n",
     "\n",
     "# This does not download the entire dataset, just the metadata and attributes describing the content. \n",

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-2Subsetting.ipynb

@@ -922,7 +922,7 @@
     "from IPython.display import display  # Fancy representation of xarray objects\n",
     "import warnings\n",
     "warnings.simplefilter('ignore')\n",
-    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n",
+    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"  # TEST_USE_PROD_DATA\n",
     "\n",
     "# Create Catalog\n",
     "cat = TDSCatalog(url)\n",
@@ -3240,7 +3240,7 @@
     }
    ],
    "source": [
-    "testurl = 'https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/ouranos/climex/QC11d3_CCCma-CanESM2_rcp85/day/historical-r1-r1i1p1-rcp85/day_tasmin_historical-r1-r1i1p1-rcp85_kda_209904_se.nc'\n",
+    "testurl = 'https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/ouranos/climex/QC11d3_CCCma-CanESM2_rcp85/day/historical-r1-r1i1p1-rcp85/day_tasmin_historical-r1-r1i1p1-rcp85_kda_209904_se.nc'  # TEST_USE_PROD_DATA\n",
     "ds = xr.open_dataset(testurl, chunks=dict(time=1))\n",
     "fig = plt.figure(figsize=(20,6))\n",
     "ax1 = plt.subplot(1,2,1)\n",

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb

@@ -777,7 +777,7 @@
     "from dask.diagnostics import ProgressBar\n",
     "warnings.simplefilter('ignore')\n",
     "# TODO change address\n",
-    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n",
+    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"  # TEST_USE_PROD_DATA\n",
     "\n",
     "# Create Catalog\n",
     "cat = TDSCatalog(url)\n",
@@ -834,6 +834,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "freq = 'YS'\n",
     "print(f'calculating yearly output using freq=\"{freq}\"')\n",
     "with ProgressBar():\n",
@@ -1583,6 +1585,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "with xclim.set_options(metadata_locales=['fr']):\n",
     "    out_fr = atmos.tx_days_above(tasmax=ds.tasmax,\n",
     "                                thresh = '27 degC', \n",
@@ -1634,7 +1638,7 @@
     "from IPython.display import clear_output, display\n",
     "\n",
     "# cb-oura-1.0 thredds url\n",
-    "url = \"http://pavics.ouranos.ca:8083/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n",
+    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"  # TEST_USE_PROD_DATA\n",
     "\n",
     "# specify output folder for netcdf files\n",
     "output = '/notebook_dir/writable-workspace/tmp/tutorial3/output'\n",

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-4Ensembles.ipynb

@@ -570,6 +570,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "from xclim import ensembles as xens\n",
     "from clisops.core import subset\n",
     "from pathlib import Path\n",
@@ -1134,6 +1136,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "# Calculate percentiles\n",
     "ds_perc = xens.ensemble_percentiles(ds_ens, values=[10, 50, 90], split=False)\n",
     "display(ds_perc)\n",

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb

@@ -16500,6 +16500,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "from xclim import ensembles as xens\n",
     "import pandas as pd\n",
     "from clisops.core import subset\n",
@@ -17206,6 +17208,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "# 30 year means and delta calculations\n",
     "window = 30\n",
     "d30yAvg = ds_ens.rolling(year=window).mean()\n",
@@ -17663,6 +17667,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "rolling = pn.widgets.FloatInput(value=30, step=10, start=10, end=30, width=50)\n",
     "rolling1 = pn.Column(pn.pane.Markdown('Smoothing'),rolling)\n",
     "\n",

content/notebooks/climate_indicators/setup_dirlayout.sh

@@ -0,0 +1,27 @@
+#!/bin/sh -x
+
+# Stable interface for the Jenkins testsuite to create any folders/files layout
+# for the notebooks to be tested under Jenkins.  With this stable interface,
+# Jenkins will not need to be modified for any future layout changes.
+#
+# Assume to be running inside the Jupyter env.
+
+THIS_FILE="`realpath "$0"`"
+THIS_DIR="`dirname "$THIS_FILE"`"
+
+# Unzip notebook 3 output to avoid having to generate it during automated
+# testing since it takes a long time.  Notebook 4 and 5 depend on this output.
+NOTEBOOK_3_OUTDIR_BASE="/notebook_dir/writable-workspace/tmp/tutorial3"
+mkdir -p $NOTEBOOK_3_OUTDIR_BASE
+unzip $THIS_DIR/output.zip -d $NOTEBOOK_3_OUTDIR_BASE/
+
+# Re-create /notebook_dir/pavics-homepage/tutorial_data layout for:
+# DriverError: /notebook_dir/pavics-homepage/tutorial_data/test_regions.geojson: No such file or directory
+#
+# Path to those .geojson files are hardcoded so users can copy the nb to
+# writable-workspace/ dir and still be able to run them seemlessly from
+# the Jupyter env (without having to also copy those *.geojson files with
+# the notebooks).
+HOMEPAGE_ROOT_DIR="/notebook_dir/pavics-homepage"
+mkdir -p $HOMEPAGE_ROOT_DIR
+ln -sv $THIS_DIR/tutorial_data $HOMEPAGE_ROOT_DIR/tutorial_data