Jenkins: allow to programmatically override all params in Jenkinsfile (…

…#136)

This new `CONFIG_PARAMETERS_SCRIPT_URL` is different than the previous
`CONFIG_OVERRIDE_SCRIPT_URL` in that it overrides all the Jenkins params
much sooner so all regular processing can take place.

When the regular processing are not enough and we need additional
customizations, then we use the other `CONFIG_OVERRIDE_SCRIPT_URL`.

The two override scripts complement each other by hooking into the
process at different moment in time, and together they allow full
complete override of the various options for the test run.

Furthermore, `CONFIG_OVERRIDE_SCRIPT_URL` can also be a local file that
is created on the fly by `CONFIG_PARAMETERS_SCRIPT_URL` so in the end,
only `CONFIG_PARAMETERS_SCRIPT_URL` is needed for a full complete
override.

Jenkinsfile

@@ -16,6 +16,8 @@ pipeline {
     }
 
     parameters {
+        string(name: 'CONFIG_PARAMETERS_SCRIPT_URL', defaultValue: '',
+               description: 'Url to a script that will be sourced, allowing to programmatically set ALL Jenkins params on this page. Ex: https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/master/test-override/jenkins-params-raven-nb-only.include.sh', trim: true)
         string(name: 'PAVICS_HOST', defaultValue: default_pavics_host,
                description: 'PAVICS host to run notebooks against.', trim: true)
         // TEST_MAGPIE_AUTH enables the evaluation of end-2-end access to some secured Thredds and Geoserver resources
@@ -92,7 +94,7 @@ Requires 'weaver' component to be active on the target 'PAVICS_HOST' server
         string(name: 'EXTRA_TEST_ENV_VAR', defaultValue: '',
                description: 'Extra environment variables for the various tests, ex: "TEST_RUNS=50 TEST_WPS_BIRDS=finch,raven,flyingpigeon TEST_NO_USE_PROD_DATA=1"', trim: true)
         string(name: 'CONFIG_OVERRIDE_SCRIPT_URL', defaultValue: '',
-               description: 'Url to a script that will be sourced, allowing to programmatically override ALL configs. Ex: https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/master/test-override/geoserver-nb-only.include.sh', trim: true)
+               description: 'Url or local file path to a script that will be sourced, allowing to programmatically override additional configs right before testsuite starts. Ex: https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/master/test-override/geoserver-nb-only.include.sh', trim: true)
         booleanParam(name: 'TEST_LOCAL_NOTEBOOKS', defaultValue: true,
                      description: 'Check the box to test notebooks in this repo (./notebooks/*.ipynb).')
         booleanParam(name: 'VERIFY_SSL', defaultValue: true,

runtest

@@ -43,11 +43,24 @@ if [ -n "$CONFIG_OVERRIDE_SCRIPT_URL" ]; then
     TMP_CONF_OVERRIDE="/tmp/conf_override"
     rm -vf "$TMP_CONF_OVERRIDE"
 
-    # Use tee to log content of override script for traceability.
-    curl --silent "$CONFIG_OVERRIDE_SCRIPT_URL" | tee "$TMP_CONF_OVERRIDE"
+
+    if echo "$CONFIG_OVERRIDE_SCRIPT_URL" | grep -e ^http ; then
+        # Use tee to log content of override script for traceability.
+        curl --silent "$CONFIG_OVERRIDE_SCRIPT_URL" | tee "$TMP_CONF_OVERRIDE"
+    else
+        # Not starting with http, it's a local file.
+        # Local file can be previously created by CONFIG_PARAMETERS_SCRIPT_URL.
+        # See example in test-override/jenkins-params-raven-specific-nb.include.sh.
+        TMP_CONF_OVERRIDE="$CONFIG_OVERRIDE_SCRIPT_URL"
+
+        # Log content of override script for traceability.
+        cat "$TMP_CONF_OVERRIDE"
+    fi
 
     # Source script so it can alter ALL existing variables in the current context.
-    . "$TMP_CONF_OVERRIDE"
+    if [ -f "$TMP_CONF_OVERRIDE" ]; then
+        . "$TMP_CONF_OVERRIDE"
+    fi
 fi
 
 py.test --nbval $NOTEBOOKS --nbval-sanitize-with notebooks/output-sanitize.cfg $PYTEST_EXTRA_OPTS

test-override/geoserver-nb-only.include.sh

@@ -1,7 +1,8 @@
 #!/bin/sh
 # Sample config override script to only run notebooks that hits GeoServer.
+# Intended for param CONFIG_OVERRIDE_SCRIPT_URL.
 
-# Have to be used together with TEST_NO_USE_PROD_DATA=1.
+# Have to be used together with TEST_NO_USE_PROD_DATA=1 (via EXTRA_TEST_ENV_VAR).
 
 NEW_NB_LIST=""
 for nb in $NOTEBOOKS; do

test-override/jenkins-params-geoserver-nb-only.include.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Sample Jenkins params override script to only run notebooks that hit GeoServer.
+# Intended for param CONFIG_PARAMETERS_SCRIPT_URL.
+
+# Enable all repos so we can filter all for them for notebooks that hit GeoServer.
+TEST_PAVICS_SDI_REPO="true"
+TEST_FINCH_REPO="true"
+TEST_PAVICS_LANDING_REPO="true"
+TEST_LOCAL_NOTEBOOKS="true"
+TEST_RAVEN_REPO="true"
+TEST_RAVENPY_REPO="true"
+
+# Force use the local GeoServer, to test the local GeoServer.  Otherwise the
+# production GeoServer will be used, which defeat the purpose of the test.
+# Need to 'export' because TEST_NO_USE_PROD_DATA was meant to be used in
+# EXTRA_TEST_ENV_VAR Jenkins build param.  TEST_NO_USE_PROD_DATA is not
+# directly one of Jenkins build param.  Build params do not need 'export'.
+export TEST_NO_USE_PROD_DATA=1
+
+# Chain with the other override script to filter the notebooks that hit
+# GeoServer only.
+CONFIG_OVERRIDE_SCRIPT_URL="https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/master/test-override/geoserver-nb-only.include.sh"
+
+# Set different test branch if required.
+#PAVICS_SDI_BRANCH=""
+#FINCH_BRANCH=""
+#PAVICS_LANDING_BRANCH=""
+#RAVEN_BRANCH=""
+#RAVENPY_BRANCH=""

test-override/jenkins-params-raven-nb-only.include.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+# Sample Jenkins params override script to only run Raven notebooks
+# with the proper --nbval-lax switch.
+# Intended for param CONFIG_PARAMETERS_SCRIPT_URL.
+
+# Disable default repos.
+TEST_PAVICS_SDI_REPO="false"
+TEST_FINCH_REPO="false"
+TEST_PAVICS_LANDING_REPO="false"
+TEST_LOCAL_NOTEBOOKS="false"
+
+# Enable raven repos.
+TEST_RAVEN_REPO="true"
+TEST_RAVENPY_REPO="true"
+
+# Raven nbs outputs are not fully up-to-date or regexed escaped properly like
+# the other default nbs so need --nbval-lax for the moment.
+PYTEST_EXTRA_OPTS="$PYTEST_EXTRA_OPTS --nbval-lax"
+
+# Set different test branch if required.
+#RAVEN_BRANCH=""
+#RAVENPY_BRANCH=""

test-override/jenkins-params-raven-specific-nb.include.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+# Sample Jenkins params override script to demonstrate on-the-fly
+# CONFIG_OVERRIDE_SCRIPT_URL file creation.
+# This script is intended for param CONFIG_PARAMETERS_SCRIPT_URL.
+
+# Scenario: we only want to run 2 specific Raven notebooks:
+# * 10_Data_assimilation.ipynb
+# * 11_Climatological_ESP_forecasting.ipynb
+
+# The beginning is same as jenkins-params-raven-nb-only.include.sh so just
+# re-use it.
+
+# Log content of override script for traceability.
+cat test-override/jenkins-params-raven-nb-only.include.sh
+
+. test-override/jenkins-params-raven-nb-only.include.sh
+
+# Then we create CONFIG_OVERRIDE_SCRIPT_URL file on-the-fly to filter for the 2
+# notebooks that we want.
+
+CONFIG_OVERRIDE_SCRIPT_URL="/tmp/specific-raven-nb.include.sh"
+
+echo '
+#!/bin/sh
+# Sample config override script to only run 2 specific raven notebooks.
+
+NEW_NB_LIST=""
+for nb in $NOTEBOOKS; do
+    if echo "$nb" | grep -i -e 10_Data_assimilation -e 11_Climatological_ESP_forecasting; then
+        NEW_NB_LIST="$NEW_NB_LIST $nb"
+    fi
+done
+NOTEBOOKS="$NEW_NB_LIST"
+' > "$CONFIG_OVERRIDE_SCRIPT_URL"

testall

@@ -4,35 +4,66 @@
 
 set -x
 
+# Allow full override of ALL Jenkins params before running test suite.
+# Intended to overrride all params in Jenkinsfile.
+#
+# This `CONFIG_PARAMETERS_SCRIPT_URL` is different than the
+# `CONFIG_OVERRIDE_SCRIPT_URL` in that it overrides all the
+# Jenkins params much sooner so all regular processing can take place.
+#
+# When the regular processing are not enough and we need additional
+# customizations, then we use the other `CONFIG_OVERRIDE_SCRIPT_URL`.
+#
+# The two override scripts complement each other by hooking into the
+# process at different moment in time, and together they allow full
+# complete override of the various options for the test run.
+#
+# Furthermore, CONFIG_OVERRIDE_SCRIPT_URL can also be a local file that is
+# created on the fly by CONFIG_PARAMETERS_SCRIPT_URL.  So in the end, only
+# CONFIG_PARAMETERS_SCRIPT_URL is needed for a full complete override.
+# See example in test-override/jenkins-params-raven-specific-nb.include.sh.
+if [ -n "$CONFIG_PARAMETERS_SCRIPT_URL" ]; then
+    TMP_PARAMS_OVERRIDE="/tmp/jenkins_params_override"
+    rm -vf "$TMP_PARAMS_OVERRIDE"
+
+    # Use tee to log content of override script for traceability.
+    curl --silent "$CONFIG_PARAMETERS_SCRIPT_URL" | tee "$TMP_PARAMS_OVERRIDE"
+
+    # Source script so it can alter ALL existing variables in the current context.
+    . "$TMP_PARAMS_OVERRIDE"
+fi
+
 # emulate "clean after checkout" of single branch pipeline
 git clean -fdx
 
 # download all additional repos containing extra notebooks to test
 ./downloadrepos
 
+# 'export' useful vars so they can be used by the CONFIG_OVERRIDE_SCRIPT_URL in runtest.
+
 # replace all slash (/) by dash (-) because (/) is illegal in folder name
 # for branch name of the format "feature/my_wizbang-feature"
 # github does the same when downloading repo archive by downloadrepos above
-PAVICS_SDI_BRANCH="`echo "$PAVICS_SDI_BRANCH" | sed "s@/@-@g"`"
-PAVICS_SDI_REPO_NAME="`echo "$PAVICS_SDI_REPO" | sed "s@^.*/@@g"`"
-FINCH_BRANCH="`echo "$FINCH_BRANCH" | sed "s@/@-@g"`"
-FINCH_REPO_NAME="`echo "$FINCH_REPO" | sed "s@^.*/@@g"`"
-PAVICS_LANDING_BRANCH="`echo "$PAVICS_LANDING_BRANCH" | sed "s@/@-@g"`"
-PAVICS_LANDING_REPO_NAME="`echo "$PAVICS_LANDING_REPO" | sed "s@^.*/@@g"`"
-RAVEN_BRANCH="`echo "$RAVEN_BRANCH" | sed "s@/@-@g"`"
-RAVEN_REPO_NAME="`echo "$RAVEN_REPO" | sed "s@^.*/@@g"`"
-RAVENPY_BRANCH="`echo "$RAVENPY_BRANCH" | sed "s@/@-@g"`"
-RAVENPY_REPO_NAME="`echo "$RAVENPY_REPO" | sed "s@^.*/@@g"`"
-ESGF_COMPUTE_API_BRANCH="`echo "$ESGF_COMPUTE_API_BRANCH" | sed "s@/@-@g"`"
-ESGF_COMPUTE_API_REPO_NAME="`echo "$ESGF_COMPUTE_API_REPO" | sed "s@^.*/@@g"`"
+export PAVICS_SDI_BRANCH="`echo "$PAVICS_SDI_BRANCH" | sed "s@/@-@g"`"
+export PAVICS_SDI_REPO_NAME="`echo "$PAVICS_SDI_REPO" | sed "s@^.*/@@g"`"
+export FINCH_BRANCH="`echo "$FINCH_BRANCH" | sed "s@/@-@g"`"
+export FINCH_REPO_NAME="`echo "$FINCH_REPO" | sed "s@^.*/@@g"`"
+export PAVICS_LANDING_BRANCH="`echo "$PAVICS_LANDING_BRANCH" | sed "s@/@-@g"`"
+export PAVICS_LANDING_REPO_NAME="`echo "$PAVICS_LANDING_REPO" | sed "s@^.*/@@g"`"
+export RAVEN_BRANCH="`echo "$RAVEN_BRANCH" | sed "s@/@-@g"`"
+export RAVEN_REPO_NAME="`echo "$RAVEN_REPO" | sed "s@^.*/@@g"`"
+export RAVENPY_BRANCH="`echo "$RAVENPY_BRANCH" | sed "s@/@-@g"`"
+export RAVENPY_REPO_NAME="`echo "$RAVENPY_REPO" | sed "s@^.*/@@g"`"
+export ESGF_COMPUTE_API_BRANCH="`echo "$ESGF_COMPUTE_API_BRANCH" | sed "s@/@-@g"`"
+export ESGF_COMPUTE_API_REPO_NAME="`echo "$ESGF_COMPUTE_API_REPO" | sed "s@^.*/@@g"`"
 
 # branches that have allowed characters such as '+' other than alphanum, '-' and '_' are converted to '-' in archives
-PAVICS_SDI_DIR=`echo "${PAVICS_SDI_REPO_NAME}-${PAVICS_SDI_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
-FINCH_DIR=`echo "${FINCH_REPO_NAME}-${FINCH_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
-PAVICS_LANDING_DIR=`echo "${PAVICS_LANDING_REPO_NAME}-${PAVICS_LANDING_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
-RAVEN_DIR=`echo "${RAVEN_REPO_NAME}-${RAVEN_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
-RAVENPY_DIR=`echo "${RAVENPY_REPO_NAME}-${RAVENPY_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
-ESGF_COMPUTE_API_DIR=`echo "${ESGF_COMPUTE_API_REPO_NAME}-${ESGF_COMPUTE_API_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
+export PAVICS_SDI_DIR=`echo "${PAVICS_SDI_REPO_NAME}-${PAVICS_SDI_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
+export FINCH_DIR=`echo "${FINCH_REPO_NAME}-${FINCH_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
+export PAVICS_LANDING_DIR=`echo "${PAVICS_LANDING_REPO_NAME}-${PAVICS_LANDING_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
+export RAVEN_DIR=`echo "${RAVEN_REPO_NAME}-${RAVEN_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
+export RAVENPY_DIR=`echo "${RAVENPY_REPO_NAME}-${RAVENPY_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
+export ESGF_COMPUTE_API_DIR=`echo "${ESGF_COMPUTE_API_REPO_NAME}-${ESGF_COMPUTE_API_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
 
 # lowercase VERIFY_SSL string
 VERIFY_SSL="`echo "$VERIFY_SSL" | tr '[:upper:]' '[:lower:]'`"