Merge branch 'master' into move-canarie-api-template-in-cowbird

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.26.4
+current_version = 1.26.9
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2023-06-06T18:08:27Z
+search = {current_version} 2023-07-04T12:44:10Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2023-06-06T18:08:27Z
+values = 2023-07-04T12:44:10Z
 
 [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

CHANGES.md

@@ -23,6 +23,86 @@
  configuration has to be moved to ensure that canarie-api configuration files aren't unintentionally mounted
  to a container that is just running an nginx proxy.
 
+[1.26.9](https://github.com/bird-house/birdhouse-deploy/tree/1.26.9) (2023-07-04)
+------------------------------------------------------------------------------------------------------------------
+
+## Fixes
+
+- Fix Cowbird's `sync_permissions` config which used invalid Magpie service types.
+
+[1.26.8](https://github.com/bird-house/birdhouse-deploy/tree/1.26.8) (2023-06-22)
+------------------------------------------------------------------------------------------------------------------
+
+## Fixes
+- Tests: some tests fail to run when `CWD` is not `COMPOSE_DIR`
+
+ The root cause is the automatic `COMPOSE_DIR` detection in
+ `read-configs.include.sh` missed one case and the detection ordering was wrong
+ for one other case as well.
+
+ This was not found before because the checkout was properly named
+ "birdhouse-deploy". When the checkout is named something else, then we hit
+ this error.
+
+ Fixes the error found here
+ https://github.com/bird-house/birdhouse-deploy/pull/329#pullrequestreview-1480211502
+
+## Changes
+- Autodeploy: document test procedure
+
+- Dev environment: add Conda `environment-dev.yml` to easily install all the dev tools required
+
+- Tests: make test runs more robust, able to run from any `CWD`
+
+ Before, test runs can only be started from inside the checkout, at some
+ "popular" locations inside the checkout. Now it can be started from
+ litterally anywhere.
+
+
+[1.26.7](https://github.com/bird-house/birdhouse-deploy/tree/1.26.7) (2023-06-19)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- A new endpoint `/services` is added that provides a JSON string describing each of the user facing services currently 
+ enabled on the stack. This is a static string and serves a different purpose than the endpoints served by canarie-api
+ (monitoring status). This endpoint is meant to be polled by the node registry scripts 
+ (https://github.com/DACCS-Climate/DACCS-node-registry) to provide information about what services are meant to be 
+ available without having to poll other endpoints directly.
+
+- A new endpoint `/version` is added that provides a string containing the current version number of the stack 
+ (e.g. "1.26.0"). This endpoint is meant to be polled by the node registry scripts 
+ (https://github.com/DACCS-Climate/DACCS-node-registry).
+
+
+[1.26.6](https://github.com/bird-house/birdhouse-deploy/tree/1.26.6) (2023-06-16)
+------------------------------------------------------------------------------------------------------------------
+
+## Fixes
+- `components/` endpoint displays intended information after auto-deploy
+
+ Previously, the script that generates the content for the `components/` endpoint
+ was using a feature of `grep` that is not supported by all versions of `grep`.
+ This meant that this script running in the auto-deployment docker container was
+ not able to properly parse the running components using `grep`. 
+ This fixes the issue by making the script compliant with all versions of `grep`.
+
+ Resolves https://github.com/bird-house/birdhouse-deploy/issues/342
+
+[1.26.5](https://github.com/bird-house/birdhouse-deploy/tree/1.26.5) (2023-06-16)
+------------------------------------------------------------------------------------------------------------------
+
+## Fixes
+- Autodeploy: optionally fix file permissions
+
+ The autodeploy mechanism creates new files owned by root. If this is not desired then users have to manually
+ update the file ownership after each autodeployment. This adds an option to change the ownership of all files
+ to a specific user after each autodeployment. 
+
+ For example, if the code in this repo is currently owned by a user named `birduser` with uid 1002, then by
+ setting `export AUTODEPLOY_CODE_OWNERSHIP="1002:1002"` in `env.local`, all files and folders in this repo will 
+ continue to be owned by `birduser` after each autodeployment. 
+
 [1.26.4](https://github.com/bird-house/birdhouse-deploy/tree/1.26.4) (2023-06-06)
 ------------------------------------------------------------------------------------------------------------------
 

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL := bash
 override APP_NAME := birdhouse-deploy
-override APP_VERSION := 1.26.4
+override APP_VERSION := 1.26.9
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -14,13 +14,13 @@ for a full-fledged production platform.
  * - releases
  - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.4.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.9.svg
  :alt: Commits since latest release
- :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.4...master
+ :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.9...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.4-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.9-blue.svg?style=flat
  :alt: Latest Tag
- :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.4
+ :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.9
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
  :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-1.26.4 2023-06-06T18:08:27Z
+1.26.9 2023-07-04T12:44:10Z

birdhouse/README.rst

@@ -262,6 +262,10 @@ To run the tests:
  python3 -m pip install -r tests/requirements.txt
  pytest tests/
 
+Some tests require internet access (to access JSON schemas used to validate
+JSON structure). If you need to run tests offline, you can skip the tests that
+require internet access by using the `-k 'not online'` pytest option.
+
 
 Tagging policy
 --------------

birdhouse/components/README.rst

@@ -135,6 +135,147 @@ Features missing in old install scripts or how the new mechanism improves on the
  via ``env.local``, which should be source controlled, meaning all surrounding maintenance related tasks can also be
  traceable and reproducible.
 
+How to test platform autodeploy is not broken by a PR
+-----------------------------------------------------
+
+There are 2 tests that need to be performed:
+
+* Can autodeploy deploy the PR from ``master`` branch, the stable reference point?
+
+ * This could fail if some changes in the PR are incompatible with autodeploy. For example: ``./pavics-compose.sh`` calls some binaries that do not exist in the autodeploy docker image.
+
+* Can autodeploy be triggered again successfully, after the PR is live?
+
+ * This could fail if the PR renamed some files and forgot to add the old file names to a ``.gitignore`` file. Then old file names will appear as new uncommitted files and autodeploy will halt because it expects a clean working directory.
+
+Here is a sample setup to test autodeploy:
+
+* Have 2 checkout directories. One is for starting the stack using ``./pavics-compose.sh``, the other one is to push new bogus changes to trigger the autodeploy mechanism.
+
+.. code-block:: shell
+
+ # this one for running pavics-compose.sh
+ git clone [email protected]:bird-house/birdhouse-deploy.git birdhouse-deploy
+
+ # this one for triggering autodeploy
+ git clone [email protected]:bird-house/birdhouse-deploy.git birdhouse-deploy-trigger
+
+* Set ``AUTODEPLOY_PLATFORM_FREQUENCY`` in ``env.local`` to a very frequent value so you do not have to wait too long for autodeploy to trigger.
+
+.. code-block:: shell
+
+ # go to the main checkout
+ cd birdhouse-deploy/birdhouse
+
+ # ensure the scheduler component is enabled, otherwise autodeploy will not work
+ echo 'export EXTRA_CONF_DIRS="$EXTRA_CONF_DIRS ./components/scheduler" >> env.local
+
+ # set AUTODEPLOY_PLATFORM_FREQUENCY
+ # can set to more frequent than 5 minutes if your machine is capable enough
+ echo 'export AUTODEPLOY_PLATFORM_FREQUENCY="@every 5m"' >> env.local
+
+ # if scheduler container already running:
+ # recreate scheduler container for new AUTODEPLOY_PLATFORM_FREQUENCY to be effective
+ ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d
+
+ # if scheduler container not running yet: start the newly added scheduler component
+ ./pavics-compose.sh up -d
+
+* Create a ``${USER}-test`` branch so you can add bogus commits without affecting your real PR. Set up your main checkout (birdhouse-deploy) to track that test branch so it will detect new changes on the test branch and trigger the autodeploy.
+
+.. code-block:: shell
+
+ # go to the main checkout
+ cd birdhouse-deploy/birdhouse
+
+ # initially create the ${USER}-test branch from master
+ # the ${USER} prefix is to avoid name clash if another user is also testing autodeploy
+ git checkout master
+ git pull
+ git checkout -b ${USER}-test
+ git push -u ${USER}-test
+
+ # ensure your runnings code is at "master" and is working correctly
+ # if you do not have a working baseline, you will not know if the breakage is due to autodeploy or your code
+ ./pavics-compose.sh up -d
+
+* Test scenario 1, from ``master`` to your PR
+
+.. code-block:: shell
+
+ # go to the other checkout to trigger autodeploy
+ cd birdhouse-deploy-trigger/birdhouse
+
+ # set branch ${USER}-test to the same commit as your PR, this will trigger autodeploy from master to your PR
+ git pull
+ git checkout ${USER}-test
+ git reset --hard YOUR_PR_BRANCH
+ git push
+
+ # now that the remote "${USER}-test" branch differs from the local "${USER}-test" branch in the birdhouse-deploy repo,
+ # the autodeploy mechanism will detect that the remote branch has changed and attempt to update the local branch
+
+ # follow logs, check for errors
+ tail -f /var/log/PAVICS/autodeploy.log
+
+ # each autodeploy trigger will start the log with
+ # ==========
+ # triggerdeploy START_TIME=2023-06-15T05:07:01+0000
+
+ # each autodeploy trigger will end the log with
+ # triggerdeploy finished START_TIME=2023-06-15T05:07:01+0000
+ # triggerdeploy finished END_TIME=2023-06-15T05:07:06+0000
+
+ # do spot checks in the log, run Jenkins on your deployment if needed
+
+* Test scenario 2, from your PR to later changes
+
+.. code-block:: shell
+
+ # go to the other checkout to trigger autodeploy
+ cd birdhouse-deploy-trigger/birdhouse
+
+ # add any bogus commit to trigger autodeploy again
+ echo >> README.rst
+ git add README.rst
+ git commit -m "trigger autodeploy"
+ git push
+
+ # now that the remote "${USER}-test" branch differs from the local "${USER}-test" branch in the birdhouse-deploy repo,
+ # the autodeploy mechanism will detect that the remote branch has changed and attempt to update the local branch
+
+ # follow logs, check for errors
+ tail -f /var/log/PAVICS/autodeploy.log
+
+* Test done, clean up the bogus ``${USER}-test`` branch and optionally relax ``AUTODEPLOY_PLATFORM_FREQUENCY``
+
+.. code-block:: shell
+
+ # go to the other checkout to trigger autodeploy
+ cd birdhouse-deploy-trigger/birdhouse
+
+ # go to master so we can delete the ${USER}-test branch
+ git checkout master
+ git push origin --delete ${USER}-test
+ git branch -D ${USER}-test
+
+ # go to the main checkout
+ cd birdhouse-deploy/birdhouse
+
+ # go to YOUR_PR_BRANCH so we can delete the ${USER}-test branch
+ git checkout YOUR_PR_BRANCH
+ git branch -D ${USER}-test
+
+ # edit env.local and change AUTODEPLOY_PLATFORM_FREQUENCY to something less frequent to save your cpu
+ # do not remove the scheduler component from the stack yet or the next command will fail
+
+ # recreate scheduler container for new AUTODEPLOY_PLATFORM_FREQUENCY to be effective
+ ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d
+
+ # optionally edit env.local to remove the scheduler component from the stack
+ # then remove the running scheduler container
+ ./pavics-compose.sh up -d --remove-orphans
+
 
 Monitoring
 ==========

birdhouse/components/cowbird/config/cowbird/config.yml.template

@@ -41,8 +41,8 @@ sync_permissions:
  # For more info on the services available in Magpie :
  # https://pavics-magpie.readthedocs.io/en/latest/services.html#available-services
  # https://pavics-magpie.readthedocs.io/en/latest/autoapi/magpie/services/index.html
- services: # Contains the different resources that can be synchronized, ordered by service
- thredds: # Service name, which should also exist in Magpie
+ services: # Contains the different resources that can be synchronized, ordered by service type
+ thredds: # Service type, which should also exist in Magpie
  # Resource key (ex.: thredds_workspace): Custom name to represent a resource path.
  #
  # Example of resource that uses variables and a `MULTI_TOKEN`.
@@ -107,7 +107,7 @@ sync_permissions:
  - "geoserver_workspace : createStoredQuery <-> thredds_workspace : write"
  weaver_outputs:
  services:
- weaver:
+ api:
  process_description:
  - name: weaver
  type: service
@@ -153,8 +153,7 @@ sync_permissions:
  type: route
  - name: "{outputID}"
  type: route
- # see 'optional-components/secure-data-proxy' for more details on protected WPS-outputs
- wps_outputs:
+ # see 'optional-components/secure-data-proxy' for more details on protected WPS-outputs
  # /wpsoutputs/weaver/{public|<user-id>}/{job-id}
  weaver_wps_outputs:
  - name: secure-data-proxy

birdhouse/components/scheduler/config.yml.template

@@ -40,6 +40,7 @@
  --env COMPOSE_DIR=${COMPOSE_DIR}
  --env AUTODEPLOY_DEPLOY_KEY_ROOT_DIR=${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}
  --env JUPYTERHUB_USER_DATA_DIR=${JUPYTERHUB_USER_DATA_DIR}
+ --env CODE_OWNERSHIP=${CODE_OWNERSHIP}
  --env AUTODEPLOY_SILENT=true${AUTODEPLOY_PLATFORM_EXTRA_DOCKER_ARGS}
  image: 'pavics/docker-compose-git:docker-18.09.7-compose-1.25.1'
 

birdhouse/components/scheduler/docker-compose-extra.yml

@@ -10,6 +10,7 @@ services:
  environment:
  COMPOSE_DIR: ${PWD}
  AUTODEPLOY_DEPLOY_KEY_ROOT_DIR: ${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}
+ CODE_OWNERSHIP: ${AUTODEPLOY_CODE_OWNERSHIP}
  restart: always
 
 # vi: tabstop=8 expandtab shiftwidth=2 softtabstop=2

birdhouse/components/weaver/.gitignore

@@ -5,6 +5,7 @@ config/weaver/wps_processes.yml
 config/weaver/weaver.ini
 config/proxy/conf.extra-service.d/weaver.conf
 config/canarie-api/canarie_api_monitoring.py
+service-config.json
 
 # Old paths. Keep these so that old config files remain uncommittable after updates.
 conf.extra-service.d/weaver.conf

birdhouse/components/weaver/service-config.json.template

@@ -0,0 +1,30 @@
+{
+ "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service",
+ "name": "weaver",
+ "keywords": [
+ "service-ogcapi_processes"
+ ],
+ "description": "An OGC-API flavored Execution Management Service",
+ "links": [
+ {
+ "rel": "service",
+ "type": "application/json",
+ "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/"
+ },
+ {
+ "rel": "service-doc",
+ "type": "text/html",
+ "href": "https://pavics-weaver.readthedocs.io/"
+ },
+ {
+ "rel": "service-desc",
+ "type": "application/json",
+ "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/"
+ },
+ {
+ "rel": "conformance",
+ "type": "application/json",
+ "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/conformance/"
+ }
+ ]
+}