Merge branch 'master' into move-canarie-api-template-in-cowbird

bird-house · Jun 13, 2023 · 0a26cd6 · 0a26cd6
2 parents ebd9091 + 22fc7c0
commit 0a26cd6
Show file tree

Hide file tree

Showing 13 changed files with 91 additions and 25 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.26.2
+current_version = 1.26.4
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2023-05-25T03:11:29Z
+search = {current_version} 2023-06-06T18:08:27Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2023-05-25T03:11:29Z
+values = 2023-06-06T18:08:27Z
 
 [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

diff --git a/CHANGES.md b/CHANGES.md
@@ -23,6 +23,44 @@
  configuration has to be moved to ensure that canarie-api configuration files aren't unintentionally mounted
  to a container that is just running an nginx proxy.
 
+[1.26.4](https://github.com/bird-house/birdhouse-deploy/tree/1.26.4) (2023-06-06)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Jupyter env: new version with latest RavenPy
+
+ See https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/pull/119 for more
+ details.
+
+
+[1.26.3](https://github.com/bird-house/birdhouse-deploy/tree/1.26.3) (2023-06-01)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Jupyter env: new version with latest Xclim and RavenPy
+
+ See https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/pull/115 for more
+ details.
+
+- Raven WPS: new version to match with new RavenPy inside the Jupyter env
+
+ See https://github.com/Ouranosinc/raven/compare/v0.14.2...v0.18.1 for more
+ details.
+
+## Fixes
+- Notebook autodeploy: unable to read the `env.local`
+
+ When `env.local` is a symlink we need to volume-mount the destination of the
+ symlink so it resolves inside the notebook autodeploy container.
+
+ This will allow notebook autodeploy config variable to be set in `env.local`.
+
+ Also had someone changed the value of `JUPYTERHUB_USER_DATA_DIR` in `env.local`,
+ it would not have worked without this fix.
+
+ This is a non-breaking fix.
+
+
 [1.26.2](https://github.com/bird-house/birdhouse-deploy/tree/1.26.2) (2023-05-25)
 ------------------------------------------------------------------------------------------------------------------
 

diff --git a/Makefile b/Makefile
@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL := bash
 override APP_NAME := birdhouse-deploy
-override APP_VERSION := 1.26.2
+override APP_VERSION := 1.26.4
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

diff --git a/README.rst b/README.rst
@@ -14,13 +14,13 @@ for a full-fledged production platform.
  * - releases
  - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.2.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.4.svg
  :alt: Commits since latest release
- :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.2...master
+ :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.4...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.2-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.4-blue.svg?style=flat
  :alt: Latest Tag
- :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.2
+ :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.4
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
  :alt: ReadTheDocs Build Status (latest version)

diff --git a/RELEASE.txt b/RELEASE.txt
@@ -1 +1 @@
-1.26.2 2023-05-25T03:11:29Z
+1.26.4 2023-06-06T18:08:27Z
diff --git a/birdhouse/components/scheduler/config.yml.template b/birdhouse/components/scheduler/config.yml.template
@@ -23,7 +23,7 @@
  --volume /tmp/notebookdeploy:/tmp/notebookdeploy:rw
  --env COMPOSE_DIR=${COMPOSE_DIR}
  --env TMP_BASE_DIR=/tmp/notebookdeploy
- --env JUPYTERHUB_USER_DATA_DIR=${JUPYTERHUB_USER_DATA_DIR}
+ --env JUPYTERHUB_USER_DATA_DIR=${JUPYTERHUB_USER_DATA_DIR}${AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS}
  image: 'docker:19.03.6-git'
 
 - name: autodeploy
@@ -40,7 +40,7 @@
  --env COMPOSE_DIR=${COMPOSE_DIR}
  --env AUTODEPLOY_DEPLOY_KEY_ROOT_DIR=${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}
  --env JUPYTERHUB_USER_DATA_DIR=${JUPYTERHUB_USER_DATA_DIR}
- --env AUTODEPLOY_SILENT=true
+ --env AUTODEPLOY_SILENT=true${AUTODEPLOY_PLATFORM_EXTRA_DOCKER_ARGS}
  image: 'pavics/docker-compose-git:docker-18.09.7-compose-1.25.1'
 
 ${AUTODEPLOY_EXTRA_SCHEDULER_JOBS}
diff --git a/birdhouse/components/scheduler/default.env b/birdhouse/components/scheduler/default.env
@@ -17,11 +17,23 @@ export AUTODEPLOY_PLATFORM_FREQUENCY="7 5 * * *"
 # Hourly
 export AUTODEPLOY_NOTEBOOK_FREQUENCY="@hourly"
 
+if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then
+ # Add volume-mount if env.local is a symlink and not a real file,
+ # so the symlink can resolve.
+ # This would likely be on a production deployment, not in a Vagrant test VM.
+ if [ -z "$(echo "$AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS" | grep "volume ${BIRDHOUSE_LOCAL_ENV_REAL_DIR}:")" ]; then
+ export AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS="$AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS
+ --volume ${BIRDHOUSE_LOCAL_ENV_REAL_DIR}:${BIRDHOUSE_LOCAL_ENV_REAL_DIR}:ro"
+ fi
+fi
+
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 OPTIONAL_VARS="
  $OPTIONAL_VARS
  \$AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES
  \$AUTODEPLOY_PLATFORM_FREQUENCY
  \$AUTODEPLOY_NOTEBOOK_FREQUENCY
  \$AUTODEPLOY_EXTRA_SCHEDULER_JOBS
+ \$AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS
+ \$AUTODEPLOY_PLATFORM_EXTRA_DOCKER_ARGS
 "
diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template
@@ -109,8 +109,8 @@ SERVICES = {
  # NOTE:
  # Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
  # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
- 'version': '1.26.2',
- 'releaseTime': '2023-05-25T03:11:29Z',
+ 'version': '1.26.4',
+ 'releaseTime': '2023-06-06T18:08:27Z',
  'institution': 'Ouranos',
  'researchSubject': 'Climatology',
  'supportEmail': '${SUPPORT_EMAIL}',
@@ -142,8 +142,8 @@ PLATFORMS = {
  # NOTE:
  # Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
  # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
- 'version': '1.26.2',
- 'releaseTime': '2023-05-25T03:11:29Z',
+ 'version': '1.26.4',
+ 'releaseTime': '2023-06-06T18:08:27Z',
  'institution': 'Ouranos',
  'researchSubject': 'Climatology',
  'supportEmail': '${SUPPORT_EMAIL}',

diff --git a/birdhouse/config/jupyterhub/default.env b/birdhouse/config/jupyterhub/default.env
@@ -8,7 +8,7 @@ export JUPYTERHUB_DOCKER=pavics/jupyterhub
 export JUPYTERHUB_VERSION=1.4.0-20210506
 
 # Jupyter single-user server images, can be overriden in env.local to have a space separated list of multiple images
-export DOCKER_NOTEBOOK_IMAGES="pavics/workflow-tests:221130-update230403"
+export DOCKER_NOTEBOOK_IMAGES="pavics/workflow-tests:230601"
 
 # Name of the image displayed on the JupyterHub image selection page
 # Can be overriden in env.local to have a space separated list of multiple images, the name order must correspond

diff --git a/birdhouse/config/raven/default.env b/birdhouse/config/raven/default.env
@@ -7,7 +7,7 @@
 # Ex: RAVEN_GEO_URL="https://${PAVICS_FQDN}/geoserver/"
 export RAVEN_GEO_URL="https://pavics.ouranos.ca/geoserver/"
 
-export RAVEN_VERSION=0.14.2
+export RAVEN_VERSION="0.18.1"
 
 # This tells the postgres service to create a database on startup
 export POSTGRES_DATABASES_TO_CREATE="

diff --git a/birdhouse/deployment/trigger-deploy-notebook b/birdhouse/deployment/trigger-deploy-notebook
@@ -29,8 +29,6 @@ START_TIME="`date -Isecond`"
 echo "==========
 notebookdeploy START_TIME=$START_TIME"
 
-set -x
-
 # running script manually (not with cron) source env.local file.
 if [ -z "$COMPOSE_DIR" ]; then
  COMPOSE_DIR="$(dirname -- "$(dirname -- "$(realpath "$0")")")"
@@ -43,6 +41,8 @@ if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then
  read_configs
 fi
 
+set -x
+
 NOTEBOOK_DIR_MNT="/notebook_dir"
 TUTORIAL_NOTEBOOKS_DIR="tutorial-notebooks"
 if [ -z "$WORKFLOW_TESTS_BRANCH" ]; then

diff --git a/birdhouse/read-configs.include.sh b/birdhouse/read-configs.include.sh
@@ -54,6 +54,20 @@ discover_compose_dir() {
 }
 
 
+discover_env_local() {
+ if [ -z "$BIRDHOUSE_LOCAL_ENV" ]; then
+ BIRDHOUSE_LOCAL_ENV="$COMPOSE_DIR/env.local"
+ fi
+
+ # env.local can be a symlink to the private config repo where the real
+ # env.local file is source controlled.
+ # Docker volume-mount will need the real dir of the file for symlink to
+ # resolve inside the container.
+ BIRDHOUSE_LOCAL_ENV_REAL_PATH="$(realpath "$BIRDHOUSE_LOCAL_ENV")"
+ BIRDHOUSE_LOCAL_ENV_REAL_DIR="$(dirname "$BIRDHOUSE_LOCAL_ENV_REAL_PATH")"
+}
+
+
 read_default_env() {
  if [ -e "$COMPOSE_DIR/default.env" ]; then
  # Ensure DELAYED_EVAL is properly initialized before being appended to.
@@ -69,7 +83,7 @@ read_default_env() {
 read_env_local() {
  # we don't use usual .env filename, because docker-compose uses it
 
- echo "Using local environment file at: ${BIRDHOUSE_LOCAL_ENV:="$COMPOSE_DIR/env.local"}"
+ echo "Using local environment file at: ${BIRDHOUSE_LOCAL_ENV}"
 
  if [ -e "$BIRDHOUSE_LOCAL_ENV" ]; then
  saved_shell_options="$(set +o)"
@@ -228,10 +242,11 @@ create_compose_conf_list() {
 # process_delayed_eval() at the appropriate moment.
 read_configs() {
  discover_compose_dir
+ discover_env_local
  read_default_env
- read_env_local # for EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS
+ read_env_local # for EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS, need discover_env_local
  read_components_default_env # uses EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS, sets ALL_CONF_DIRS
- read_env_local # again to override components default.env
+ read_env_local # again to override components default.env, need discover_env_local
  process_delayed_eval
 }
 
@@ -241,7 +256,8 @@ read_configs() {
 # read_configs() to be safe.
 read_basic_configs_only() {
  discover_compose_dir
+ discover_env_local
  read_default_env
- read_env_local
+ read_env_local # need discover_env_local
  process_delayed_eval
 }
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.26.2'
+version = '1.26.4'
 # The full version, including alpha/beta/rc tags.
-release = '1.26.2'
+release = '1.26.4'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.