From 70f85a75f2d2fd43f70a033ab5851528bbff6561 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Wed, 24 May 2023 13:46:15 -0400 Subject: [PATCH 01/39] add version and service endpoints --- CHANGES.md | 8 +++ birdhouse/components/weaver/.gitignore | 1 + .../weaver/service-config.json.template | 7 +++ birdhouse/config/finch/.gitignore | 1 + .../config/finch/service-config.json.template | 7 +++ birdhouse/config/flyingpigeon/.gitignore | 1 + .../flyingpigeon/service-config.json.template | 7 +++ birdhouse/config/geoserver/.gitignore | 1 + .../geoserver/service-config.json.template | 7 +++ birdhouse/config/hummingbird/.gitignore | 1 + .../hummingbird/service-config.json.template | 7 +++ birdhouse/config/jupyterhub/.gitignore | 1 + .../jupyterhub/service-config.json.template | 7 +++ .../conf.d/all-services.include.template | 10 ++++ birdhouse/config/proxy/default.env | 2 + birdhouse/config/raven/.gitignore | 1 + .../config/raven/service-config.json.template | 7 +++ birdhouse/config/thredds/.gitignore | 1 + .../thredds/service-config.json.template | 7 +++ birdhouse/pavics-compose.sh | 2 + .../scripts/get-services-json.include.sh | 15 +++++ birdhouse/scripts/get-version.include.sh | 9 +++ tests/requirements.txt | 1 + tests/service-config-schema.json | 34 +++++++++++ tests/test_deployment.py | 56 +++++++++++++++++++ 25 files changed, 201 insertions(+) create mode 100644 birdhouse/components/weaver/service-config.json.template create mode 100644 birdhouse/config/finch/service-config.json.template create mode 100644 birdhouse/config/flyingpigeon/service-config.json.template create mode 100644 birdhouse/config/geoserver/service-config.json.template create mode 100644 birdhouse/config/hummingbird/service-config.json.template create mode 100644 birdhouse/config/jupyterhub/service-config.json.template create mode 100644 birdhouse/config/raven/service-config.json.template create mode 100644 birdhouse/config/thredds/service-config.json.template create mode 100755 birdhouse/scripts/get-services-json.include.sh create mode 100755 birdhouse/scripts/get-version.include.sh create mode 100644 tests/service-config-schema.json create mode 100644 tests/test_deployment.py diff --git a/CHANGES.md b/CHANGES.md index 90ab6d8be..f9399a443 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,14 @@ ## Changes +- A new endpoint "/services" is added that provides a json string describing each of the user facing services currently + enabled on the stack. This is a static string and serves a different purpose than the endpoints served by canarie-api + (monitoring status). This endpoint is meant to be polled by the node registry scripts (in development) to provide + information about what services are meant to be available without having to poll other endpoints directly. + +- A new endpoint "/version" is added that provides a string containing the current version number of the stack + (e.g. "1.26.0"). This endpoint is meant to be polled by the node registry scripts (in development). + - Licence: update copyright line with year and ownership diff --git a/birdhouse/components/weaver/.gitignore b/birdhouse/components/weaver/.gitignore index be0fc46fe..1cd0a2456 100644 --- a/birdhouse/components/weaver/.gitignore +++ b/birdhouse/components/weaver/.gitignore @@ -5,6 +5,7 @@ config/weaver/wps_processes.yml config/weaver/weaver.ini config/proxy/conf.extra-service.d/weaver.conf config/canarie-api/canarie_api_monitoring.py +service-config.json # Old paths. Keep these so that old config files remain uncommittable after updates. conf.extra-service.d/weaver.conf diff --git a/birdhouse/components/weaver/service-config.json.template b/birdhouse/components/weaver/service-config.json.template new file mode 100644 index 000000000..f7e26152c --- /dev/null +++ b/birdhouse/components/weaver/service-config.json.template @@ -0,0 +1,7 @@ +{ + "name": "weaver", + "url": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}", + "type": "ems", + "documentation": "https://pavics-weaver.readthedocs.io", + "description": "An OGC-API flavored Execution Management Service" +} diff --git a/birdhouse/config/finch/.gitignore b/birdhouse/config/finch/.gitignore index ee90d845c..e6f9bf889 100644 --- a/birdhouse/config/finch/.gitignore +++ b/birdhouse/config/finch/.gitignore @@ -1,6 +1,7 @@ wps.cfg config/magpie/providers.cfg config/canarie-api/canarie_api_monitoring.py +service-config.json # Old paths. Keep these so that old config files remain uncommittable after updates. config/proxy/canarie_api_monitoring.py diff --git a/birdhouse/config/finch/service-config.json.template b/birdhouse/config/finch/service-config.json.template new file mode 100644 index 000000000..3a745a4de --- /dev/null +++ b/birdhouse/config/finch/service-config.json.template @@ -0,0 +1,7 @@ +{ + "name": "finch", + "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch", + "type": "wps", + "documentation": "https://finch.readthedocs.io", + "description": "A Web Processing Service for Climate Indicators." +} diff --git a/birdhouse/config/flyingpigeon/.gitignore b/birdhouse/config/flyingpigeon/.gitignore index 6156a69ce..2be797c77 100644 --- a/birdhouse/config/flyingpigeon/.gitignore +++ b/birdhouse/config/flyingpigeon/.gitignore @@ -2,3 +2,4 @@ wps.cfg config/canarie-api/canarie_api_monitoring.py config/magpie/permissions.cfg config/magpie/providers.cfg +service-config.json diff --git a/birdhouse/config/flyingpigeon/service-config.json.template b/birdhouse/config/flyingpigeon/service-config.json.template new file mode 100644 index 000000000..524462247 --- /dev/null +++ b/birdhouse/config/flyingpigeon/service-config.json.template @@ -0,0 +1,7 @@ +{ + "name": "flyingpigeon", + "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon", + "type": "wps", + "documentation": "https://flyingpigeon.readthedocs.io", + "description": "WPS for climate model data, indices and extreme events." +} diff --git a/birdhouse/config/geoserver/.gitignore b/birdhouse/config/geoserver/.gitignore index e179901f6..190f04012 100644 --- a/birdhouse/config/geoserver/.gitignore +++ b/birdhouse/config/geoserver/.gitignore @@ -1,6 +1,7 @@ config/proxy/conf.extra-service.d/geoserver.conf config/canarie-api/canarie_api_monitoring.py config/magpie/providers.cfg +service-config.json # Old paths. Keep these so that old config files remain uncommittable after updates. geoserver_canarie_api_monitoring.py diff --git a/birdhouse/config/geoserver/service-config.json.template b/birdhouse/config/geoserver/service-config.json.template new file mode 100644 index 000000000..6dca5a2b7 --- /dev/null +++ b/birdhouse/config/geoserver/service-config.json.template @@ -0,0 +1,7 @@ +{ + "name": "geoserver", + "url": "https://${PAVICS_FQDN_PUBLIC}/geoserver", + "type": "geoserver", + "documentation": "https://docs.geoserver.org/", + "description": "GeoServer is a server that allows users to view and edit geospatial data." +} diff --git a/birdhouse/config/hummingbird/.gitignore b/birdhouse/config/hummingbird/.gitignore index 02da491b5..cadf22493 100644 --- a/birdhouse/config/hummingbird/.gitignore +++ b/birdhouse/config/hummingbird/.gitignore @@ -1,3 +1,4 @@ config/canarie-api/canarie_api_monitoring.py config/magpie/providers.cfg custom.cfg +service-config.json diff --git a/birdhouse/config/hummingbird/service-config.json.template b/birdhouse/config/hummingbird/service-config.json.template new file mode 100644 index 000000000..6d56d4de7 --- /dev/null +++ b/birdhouse/config/hummingbird/service-config.json.template @@ -0,0 +1,7 @@ +{ + "name": "hummingbird", + "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird", + "type": "wps", + "documentation": "https://birdhouse-hummingbird.readthedocs.io", + "description": "A WPS for metadata compliance checks used in the climate science community." +} diff --git a/birdhouse/config/jupyterhub/.gitignore b/birdhouse/config/jupyterhub/.gitignore index 439c8b34b..6563f9518 100644 --- a/birdhouse/config/jupyterhub/.gitignore +++ b/birdhouse/config/jupyterhub/.gitignore @@ -2,6 +2,7 @@ custom_templates/login.html jupyterhub_config.py config/proxy/conf.extra-service.d/jupyterhub.conf config/canarie-api/canarie_api_monitoring.py +service-config.json # Old paths. Keep these so that old config files remain uncommittable after updates. jupyterhub_canarie_api_monitoring.py diff --git a/birdhouse/config/jupyterhub/service-config.json.template b/birdhouse/config/jupyterhub/service-config.json.template new file mode 100644 index 000000000..5adf6acbd --- /dev/null +++ b/birdhouse/config/jupyterhub/service-config.json.template @@ -0,0 +1,7 @@ +{ + "name": "jupyterhub", + "url": "https://${PAVICS_FQDN_PUBLIC}/jupyter", + "type": "jupyterhub", + "documentation": "https://jupyter.org/hub", + "description": "An interactive development environment use to create and share documents that contain live code." +} diff --git a/birdhouse/config/proxy/conf.d/all-services.include.template b/birdhouse/config/proxy/conf.d/all-services.include.template index 30283a64a..0f80f9d6a 100644 --- a/birdhouse/config/proxy/conf.d/all-services.include.template +++ b/birdhouse/config/proxy/conf.d/all-services.include.template @@ -7,6 +7,16 @@ return 200 '${BIRDHOUSE_DEPLOY_COMPONENTS_JSON}'; } + location /services { + default_type application/json; + return 200 '${BIRDHOUSE_DEPLOY_SERVICES_JSON}'; + } + + location /version { + default_type text/plain; + return 200 '${BIRDHOUSE_VERSION}'; + } + location /doc { return 302 ${DOC_URL}; } diff --git a/birdhouse/config/proxy/default.env b/birdhouse/config/proxy/default.env index aa04c73c9..7a74a2430 100644 --- a/birdhouse/config/proxy/default.env +++ b/birdhouse/config/proxy/default.env @@ -21,6 +21,8 @@ export DELAYED_EVAL=" export VARS=" $VARS \$BIRDHOUSE_DEPLOY_COMPONENTS_JSON + \$BIRDHOUSE_DEPLOY_SERVICES_JSON + \$BIRDHOUSE_VERSION " export OPTIONAL_VARS=" diff --git a/birdhouse/config/raven/.gitignore b/birdhouse/config/raven/.gitignore index ff9d09569..e894caebf 100644 --- a/birdhouse/config/raven/.gitignore +++ b/birdhouse/config/raven/.gitignore @@ -1,6 +1,7 @@ config/canarie-api/canarie_api_monitoring.py config/magpie/providers.cfg wps.cfg +service-config.json # Old paths. Keep these so that old config files remain uncommittable after updates. config/proxy/canarie_api_monitoring.py diff --git a/birdhouse/config/raven/service-config.json.template b/birdhouse/config/raven/service-config.json.template new file mode 100644 index 000000000..214d05544 --- /dev/null +++ b/birdhouse/config/raven/service-config.json.template @@ -0,0 +1,7 @@ +{ + "name": "raven", + "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven", + "type": "wps", + "documentation": "https://pavics-raven.readthedocs.io", + "description": "A WPS providing hydrological modeling and analysis services." +} diff --git a/birdhouse/config/thredds/.gitignore b/birdhouse/config/thredds/.gitignore index 1ebc49e90..bcc9b17a9 100644 --- a/birdhouse/config/thredds/.gitignore +++ b/birdhouse/config/thredds/.gitignore @@ -2,6 +2,7 @@ catalog.xml threddsConfig.xml config/proxy/conf.extra-service.d/thredds.conf config/canarie-api/canarie_api_monitoring.py +service-config.json # Old paths. Keep these so that old config files remain uncommittable after updates. config/proxy/canarie_api_monitoring.py diff --git a/birdhouse/config/thredds/service-config.json.template b/birdhouse/config/thredds/service-config.json.template new file mode 100644 index 000000000..72b8fa67e --- /dev/null +++ b/birdhouse/config/thredds/service-config.json.template @@ -0,0 +1,7 @@ +{ + "name": "thredds", + "url": "https://${PAVICS_FQDN_PUBLIC}/thredds", + "type": "thredds", + "documentation": "https://www.unidata.ucar.edu/software/tds/", + "description": "A server that provides catalog, metadata, and data access services for scientific data." +} diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index ed37f48e8..34de012a9 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -49,6 +49,8 @@ COMPOSE_DIR="`pwd`" read_configs # this sets ALL_CONF_DIRS . ./scripts/get-components-json.include.sh +. ./scripts/get-services-json.include.sh +. ./scripts/get-version.include.sh for i in ${VARS} do diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh new file mode 100755 index 000000000..9fd6a6b9b --- /dev/null +++ b/birdhouse/scripts/get-services-json.include.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# default value in case of error or missing definitions +export BIRDHOUSE_DEPLOY_SERVICES_JSON='{}' +if [ -z "${ALL_CONF_DIRS}" ]; then + echo "No services in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. SERVICES JSON list will be empty!" + return +fi + +for adir in $(echo "$ALL_CONF_DIRS" | grep -v " ./core\| ./data"); do + [ -f "${adir}/service-config.json" ] || continue + SERVICES="${SERVICES}$([ -n "${SERVICES}" ] && echo ',') $(cat "${adir}/service-config.json")" +done + +export BIRDHOUSE_DEPLOY_SERVICES_JSON="{\"services\": [${SERVICES}]}" diff --git a/birdhouse/scripts/get-version.include.sh b/birdhouse/scripts/get-version.include.sh new file mode 100755 index 000000000..72d0a1de1 --- /dev/null +++ b/birdhouse/scripts/get-version.include.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# get the current version of this deployment +RELEASE_FILE="$(dirname "${COMPOSE_DIR}")/RELEASE.txt" + +if [ -f "${RELEASE_FILE}" ]; then + BIRDHOUSE_VERSION="$(head -1 "$RELEASE_FILE" | cut -d" " -f 1)" + export BIRDHOUSE_VERSION +fi diff --git a/tests/requirements.txt b/tests/requirements.txt index c021c5b5b..876fb2f39 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1 +1,2 @@ pytest==7.2.2 +jsonschema==4.17.3 diff --git a/tests/service-config-schema.json b/tests/service-config-schema.json new file mode 100644 index 000000000..5d8366a46 --- /dev/null +++ b/tests/service-config-schema.json @@ -0,0 +1,34 @@ +{ + "type": "object", + "required": [ + "url", + "name", + "type", + "documentation", + "description" + ], + "properties": { + "href": { + "type": "string", + "format": "uri", + "pattern": "^https?://" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "minLength": 1 + }, + "documentation": { + "type": "string", + "format": "uri", + "pattern": "^https?://" + }, + "description": { + "type": "string", + "minLength": 1 + } + } +} \ No newline at end of file diff --git a/tests/test_deployment.py b/tests/test_deployment.py new file mode 100644 index 000000000..499313226 --- /dev/null +++ b/tests/test_deployment.py @@ -0,0 +1,56 @@ +import os +import pytest +import glob +import json +from string import Template + +import jsonschema + +COMPONENT_LOCATIONS = ("core", "data", "services", "extensions", "test-helpers") +TEMPLATE_SUBSTITUTIONS = {"PAVICS_FQDN_PUBLIC": os.environ.get("PAVICS_FQDN_PUBLIC", "example.com")} + + +@pytest.fixture(scope="module") +def root_dir(request): + yield os.path.dirname(os.path.dirname(request.fspath)) + + +@pytest.fixture +def component_paths(root_dir): + yield [path for loc in COMPONENT_LOCATIONS for path in glob.glob(os.path.join(root_dir, "birdhouse", loc, "*"))] + + +@pytest.fixture(scope="module") +def services_config_schema(request): + with open(os.path.join(os.path.dirname(request.fspath), "service-config-schema.json")) as f: + return json.load(f) + + +class TestDockerCompose: + def test_service_config_name_same_as_dirname(self, component_paths): + invalid_names = [] + + for path in component_paths: + service_config_file = os.path.join(path, "service-config.json.template") + if os.path.isfile(service_config_file): + with open(service_config_file) as f: + service_config = json.loads(Template(f.read()).substitute(TEMPLATE_SUBSTITUTIONS)) + config_name = service_config.get("name") + path_name = os.path.basename(path) + if config_name != path_name: + invalid_names.append((config_name, path_name)) + assert not invalid_names, "service names in service-config.json.template should match the directory name" + + def test_service_config_valid(self, component_paths, services_config_schema): + invalid_schemas = [] + + for path in component_paths: + service_config_file = os.path.join(path, "service-config.json.template") + if os.path.isfile(service_config_file): + with open(service_config_file) as f: + service_config = json.loads(Template(f.read()).substitute(TEMPLATE_SUBSTITUTIONS)) + try: + jsonschema.validate(instance=service_config, schema=services_config_schema) + except jsonschema.exceptions.ValidationError as e: + invalid_schemas.append(f"{os.path.basename(path)} contains invalid service configuration: {e}") + assert not invalid_schemas, "\n".join(invalid_schemas) From 1d39b63c0931dc7adfa1d560fa514f92db2b2604 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Thu, 25 May 2023 15:37:00 -0400 Subject: [PATCH 02/39] minor fixes and test updates --- CHANGES.md | 12 +++++++----- .../components/weaver/service-config.json.template | 2 +- birdhouse/config/finch/service-config.json.template | 2 +- .../config/flyingpigeon/service-config.json.template | 2 +- .../config/geoserver/service-config.json.template | 2 +- .../config/hummingbird/service-config.json.template | 2 +- .../config/jupyterhub/service-config.json.template | 2 +- birdhouse/config/raven/service-config.json.template | 2 +- .../config/thredds/service-config.json.template | 2 +- birdhouse/scripts/get-services-json.include.sh | 2 +- tests/service-config-schema.json | 7 +++++-- tests/test_deployment.py | 8 ++++++-- 12 files changed, 27 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f9399a443..0f082a892 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,13 +17,15 @@ ## Changes -- A new endpoint "/services" is added that provides a json string describing each of the user facing services currently +- A new endpoint `/services` is added that provides a JSON string describing each of the user facing services currently enabled on the stack. This is a static string and serves a different purpose than the endpoints served by canarie-api - (monitoring status). This endpoint is meant to be polled by the node registry scripts (in development) to provide - information about what services are meant to be available without having to poll other endpoints directly. + (monitoring status). This endpoint is meant to be polled by the node registry scripts + (https://github.com/DACCS-Climate/DACCS-node-registry) to provide information about what services are meant to be + available without having to poll other endpoints directly. -- A new endpoint "/version" is added that provides a string containing the current version number of the stack - (e.g. "1.26.0"). This endpoint is meant to be polled by the node registry scripts (in development). +- A new endpoint `/version` is added that provides a string containing the current version number of the stack + (e.g. "1.26.0"). This endpoint is meant to be polled by the node registry scripts + (https://github.com/DACCS-Climate/DACCS-node-registry). - Licence: update copyright line with year and ownership diff --git a/birdhouse/components/weaver/service-config.json.template b/birdhouse/components/weaver/service-config.json.template index f7e26152c..20a88dc58 100644 --- a/birdhouse/components/weaver/service-config.json.template +++ b/birdhouse/components/weaver/service-config.json.template @@ -1,7 +1,7 @@ { "name": "weaver", "url": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}", - "type": "ems", + "type": ["ogc-api"], "documentation": "https://pavics-weaver.readthedocs.io", "description": "An OGC-API flavored Execution Management Service" } diff --git a/birdhouse/config/finch/service-config.json.template b/birdhouse/config/finch/service-config.json.template index 3a745a4de..275bf61b2 100644 --- a/birdhouse/config/finch/service-config.json.template +++ b/birdhouse/config/finch/service-config.json.template @@ -1,7 +1,7 @@ { "name": "finch", "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch", - "type": "wps", + "type": ["wps"], "documentation": "https://finch.readthedocs.io", "description": "A Web Processing Service for Climate Indicators." } diff --git a/birdhouse/config/flyingpigeon/service-config.json.template b/birdhouse/config/flyingpigeon/service-config.json.template index 524462247..ea04f41e1 100644 --- a/birdhouse/config/flyingpigeon/service-config.json.template +++ b/birdhouse/config/flyingpigeon/service-config.json.template @@ -1,7 +1,7 @@ { "name": "flyingpigeon", "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon", - "type": "wps", + "type": ["wps"], "documentation": "https://flyingpigeon.readthedocs.io", "description": "WPS for climate model data, indices and extreme events." } diff --git a/birdhouse/config/geoserver/service-config.json.template b/birdhouse/config/geoserver/service-config.json.template index 6dca5a2b7..ccade0e16 100644 --- a/birdhouse/config/geoserver/service-config.json.template +++ b/birdhouse/config/geoserver/service-config.json.template @@ -1,7 +1,7 @@ { "name": "geoserver", "url": "https://${PAVICS_FQDN_PUBLIC}/geoserver", - "type": "geoserver", + "type": ["geoserver"], "documentation": "https://docs.geoserver.org/", "description": "GeoServer is a server that allows users to view and edit geospatial data." } diff --git a/birdhouse/config/hummingbird/service-config.json.template b/birdhouse/config/hummingbird/service-config.json.template index 6d56d4de7..50c29659d 100644 --- a/birdhouse/config/hummingbird/service-config.json.template +++ b/birdhouse/config/hummingbird/service-config.json.template @@ -1,7 +1,7 @@ { "name": "hummingbird", "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird", - "type": "wps", + "type": ["wps"], "documentation": "https://birdhouse-hummingbird.readthedocs.io", "description": "A WPS for metadata compliance checks used in the climate science community." } diff --git a/birdhouse/config/jupyterhub/service-config.json.template b/birdhouse/config/jupyterhub/service-config.json.template index 5adf6acbd..ee4c3b6eb 100644 --- a/birdhouse/config/jupyterhub/service-config.json.template +++ b/birdhouse/config/jupyterhub/service-config.json.template @@ -1,7 +1,7 @@ { "name": "jupyterhub", "url": "https://${PAVICS_FQDN_PUBLIC}/jupyter", - "type": "jupyterhub", + "type": ["jupyterhub"], "documentation": "https://jupyter.org/hub", "description": "An interactive development environment use to create and share documents that contain live code." } diff --git a/birdhouse/config/raven/service-config.json.template b/birdhouse/config/raven/service-config.json.template index 214d05544..2453f389a 100644 --- a/birdhouse/config/raven/service-config.json.template +++ b/birdhouse/config/raven/service-config.json.template @@ -1,7 +1,7 @@ { "name": "raven", "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven", - "type": "wps", + "type": ["wps"], "documentation": "https://pavics-raven.readthedocs.io", "description": "A WPS providing hydrological modeling and analysis services." } diff --git a/birdhouse/config/thredds/service-config.json.template b/birdhouse/config/thredds/service-config.json.template index 72b8fa67e..bc93cbab4 100644 --- a/birdhouse/config/thredds/service-config.json.template +++ b/birdhouse/config/thredds/service-config.json.template @@ -1,7 +1,7 @@ { "name": "thredds", "url": "https://${PAVICS_FQDN_PUBLIC}/thredds", - "type": "thredds", + "type": ["thredds"], "documentation": "https://www.unidata.ucar.edu/software/tds/", "description": "A server that provides catalog, metadata, and data access services for scientific data." } diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh index 9fd6a6b9b..df3b6a15a 100755 --- a/birdhouse/scripts/get-services-json.include.sh +++ b/birdhouse/scripts/get-services-json.include.sh @@ -7,7 +7,7 @@ if [ -z "${ALL_CONF_DIRS}" ]; then return fi -for adir in $(echo "$ALL_CONF_DIRS" | grep -v " ./core\| ./data"); do +for adir in ${ALL_CONF_DIRS}; do [ -f "${adir}/service-config.json" ] || continue SERVICES="${SERVICES}$([ -n "${SERVICES}" ] && echo ',') $(cat "${adir}/service-config.json")" done diff --git a/tests/service-config-schema.json b/tests/service-config-schema.json index 5d8366a46..40045936c 100644 --- a/tests/service-config-schema.json +++ b/tests/service-config-schema.json @@ -18,8 +18,11 @@ "minLength": 1 }, "type": { - "type": "string", - "minLength": 1 + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } }, "documentation": { "type": "string", diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 499313226..0d576441f 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -6,8 +6,12 @@ import jsonschema -COMPONENT_LOCATIONS = ("core", "data", "services", "extensions", "test-helpers") -TEMPLATE_SUBSTITUTIONS = {"PAVICS_FQDN_PUBLIC": os.environ.get("PAVICS_FQDN_PUBLIC", "example.com")} +COMPONENT_LOCATIONS = ("components", "optional-components", "config") +TEMPLATE_SUBSTITUTIONS = { + "PAVICS_FQDN_PUBLIC": os.environ.get("PAVICS_FQDN_PUBLIC", "example.com"), + "WEAVER_MANAGER_NAME": os.environ.get("WEAVER_MANAGER_NAME", "weaver"), + "TWITCHER_PROTECTED_PATH": os.environ.get("TWITCHER_PROTECTED_PATH", "/twitcher/ows/proxy") +} @pytest.fixture(scope="module") From 796f5c59e6e8728dbd75a7d3b915743009a53c15 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 1 Jun 2023 15:19:37 -0400 Subject: [PATCH 03/39] Document how to test Autodeploy Together with the Scheduler component description where Autodeploy reside. --- birdhouse/components/README.rst | 116 ++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index e2ff2e9f1..701ee60ab 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -135,6 +135,122 @@ Features missing in old install scripts or how the new mechanism improves on the via ``env.local``, which should be source controlled, meaning all surrounding maintenance related tasks can also be traceable and reproducible. +How to test platform autodeploy is not broken by a PR +----------------------------------------------------- + +There are 2 tests that need to be performed: + +* Can autodeploy deploy the PR from ``master`` branch, the stable reference point + + * This could fail if some changes, in the PR, are compatible with autodeploy, example ``./pavics-compose.sh`` calls some binaries that do not exist in the autodeploy docker image. + +* Can autodeploy be triggered again successfully, after PR is live + + * This could fail if PR renamed some files and forgot to add to ``.gitignore`` the old file names. Then old file names will appears as new uncommitted files and autodeploy will halt because it expects a clean working directory. + +Here is a sample setup to test autodeploy: + +* Have 2 checkout directories. One is for starting the stack using ``./pavics-compose.sh``, the other one is to push new bogus change to trigger the autodeploy. + +.. code-block:: shell + + # this one for running pavics-compose.sh + git clone git@github.com:bird-house/birdhouse-deploy.git birdhouse-deploy + + # this one for triggering autodeploy + git clone git@github.com:bird-house/birdhouse-deploy.git birdhouse-deploy-trigger + +* Set ``AUTODEPLOY_PLATFORM_FREQUENCY`` in ``env.local`` to a very frequent value so you do not wait too long for autodeploy to trigger. + +.. code-block:: shell + + # go to the main checkout + cd birdhouse-deploy/birdhouse + + # ensure the scheduler component is enabled, otherwise autodeploy will not work + echo 'export EXTRA_CONF_DIRS="$EXTRA_CONF_DIRS ./components/scheduler" >> env.local + + # set AUTODEPLOY_PLATFORM_FREQUENCY + echo 'export AUTODEPLOY_PLATFORM_FREQUENCY="@every 5m"' >> env.local + + # recreate scheduler container for new AUTODEPLOY_PLATFORM_FREQUENCY to be effective + ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d + +* Create a ``test`` branch so you can add bogus commits without affecting your real PR. Setup your main checkout to track that test branch so it new changes to trigger. + +.. code-block:: shell + + # go to the main checkout + cd birdhouse-deploy/birdhouse + + # initially create the test branch from master + git checkout master + git pull + git checkout -b test + git push -u test + + # ensure your runnings code is at "master" and is working correctly + # if you do not have a working baseline, you will not know if the breakage is due to autodeploy or your code + ./pavics-compose.sh up -d + +* Test scenario 1, from ``master`` to your PR + +.. code-block:: shell + + # go to the other checkout to trigger autodeploy + cd birdhouse-deploy-trigger/birdhouse + + # set branch test to the same commit as your PR, this will trigger autodeploy from master to your PR + git pull + git checkout test + git reset --hard YOUR_PR_BRANCH + git push + + # follow logs, check for errors + tail -f /var/log/PAVICS/autodeploy.log + + # do spot checks, run Jenkins on your deployment if needed + +* Test scenario 2, from your PR and later + +.. code-block:: shell + + # go to the other checkout to trigger autodeploy + cd birdhouse-deploy-trigger/birdhouse + + # add any bogus commit to trigger autodeploy again + echo >> README.rst + git ci -am "trigger autodeploy" + git push + + # follow logs, check for errors + tail -f /var/log/PAVICS/autodeploy.log + +* Test done, clean up the bogus ``test`` branch and optionally relax AUTODEPLOY_PLATFORM_FREQUENCY + +.. code-block:: shell + + # go to the other checkout to trigger autodeploy + cd birdhouse-deploy-trigger/birdhouse + + # go to master so we can delete the test branch + git checkout master + git push origin --delete test + git branch -D test + + # go to the main checkout + cd birdhouse-deploy/birdhouse + + # go to YOUR_PR_BRANCH so we can delete the test branch + git checkout YOUR_PR_BRANCH + git push origin --delete test + git branch -D test + + # edit env.local and change AUTODEPLOY_PLATFORM_FREQUENCY to something less frequent to save your cpu + + # recreate scheduler container for new AUTODEPLOY_PLATFORM_FREQUENCY to be effective + ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d + Monitoring ========== From 7a56573107f3c4300b12c156defa7c10b2ecc914 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Thu, 1 Jun 2023 15:26:17 -0400 Subject: [PATCH 04/39] restrict service types --- birdhouse/components/weaver/service-config.json.template | 2 +- birdhouse/config/finch/service-config.json.template | 2 +- birdhouse/config/flyingpigeon/service-config.json.template | 2 +- birdhouse/config/geoserver/service-config.json.template | 2 +- birdhouse/config/hummingbird/service-config.json.template | 2 +- birdhouse/config/raven/service-config.json.template | 2 +- birdhouse/config/thredds/service-config.json.template | 2 +- tests/service-config-schema.json | 3 ++- 8 files changed, 9 insertions(+), 8 deletions(-) diff --git a/birdhouse/components/weaver/service-config.json.template b/birdhouse/components/weaver/service-config.json.template index 20a88dc58..d49ce527d 100644 --- a/birdhouse/components/weaver/service-config.json.template +++ b/birdhouse/components/weaver/service-config.json.template @@ -1,7 +1,7 @@ { "name": "weaver", "url": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}", - "type": ["ogc-api"], + "type": ["service-ogcapi"], "documentation": "https://pavics-weaver.readthedocs.io", "description": "An OGC-API flavored Execution Management Service" } diff --git a/birdhouse/config/finch/service-config.json.template b/birdhouse/config/finch/service-config.json.template index 275bf61b2..a9e3f9a8b 100644 --- a/birdhouse/config/finch/service-config.json.template +++ b/birdhouse/config/finch/service-config.json.template @@ -1,7 +1,7 @@ { "name": "finch", "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch", - "type": ["wps"], + "type": ["service-wps"], "documentation": "https://finch.readthedocs.io", "description": "A Web Processing Service for Climate Indicators." } diff --git a/birdhouse/config/flyingpigeon/service-config.json.template b/birdhouse/config/flyingpigeon/service-config.json.template index ea04f41e1..0ab2c3d99 100644 --- a/birdhouse/config/flyingpigeon/service-config.json.template +++ b/birdhouse/config/flyingpigeon/service-config.json.template @@ -1,7 +1,7 @@ { "name": "flyingpigeon", "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon", - "type": ["wps"], + "type": ["service-wps"], "documentation": "https://flyingpigeon.readthedocs.io", "description": "WPS for climate model data, indices and extreme events." } diff --git a/birdhouse/config/geoserver/service-config.json.template b/birdhouse/config/geoserver/service-config.json.template index ccade0e16..d32203684 100644 --- a/birdhouse/config/geoserver/service-config.json.template +++ b/birdhouse/config/geoserver/service-config.json.template @@ -1,7 +1,7 @@ { "name": "geoserver", "url": "https://${PAVICS_FQDN_PUBLIC}/geoserver", - "type": ["geoserver"], + "type": ["data", "service-wms"], "documentation": "https://docs.geoserver.org/", "description": "GeoServer is a server that allows users to view and edit geospatial data." } diff --git a/birdhouse/config/hummingbird/service-config.json.template b/birdhouse/config/hummingbird/service-config.json.template index 50c29659d..6d44d6b4c 100644 --- a/birdhouse/config/hummingbird/service-config.json.template +++ b/birdhouse/config/hummingbird/service-config.json.template @@ -1,7 +1,7 @@ { "name": "hummingbird", "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird", - "type": ["wps"], + "type": ["service-wps"], "documentation": "https://birdhouse-hummingbird.readthedocs.io", "description": "A WPS for metadata compliance checks used in the climate science community." } diff --git a/birdhouse/config/raven/service-config.json.template b/birdhouse/config/raven/service-config.json.template index 2453f389a..0b119a8ae 100644 --- a/birdhouse/config/raven/service-config.json.template +++ b/birdhouse/config/raven/service-config.json.template @@ -1,7 +1,7 @@ { "name": "raven", "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven", - "type": ["wps"], + "type": ["service-wps"], "documentation": "https://pavics-raven.readthedocs.io", "description": "A WPS providing hydrological modeling and analysis services." } diff --git a/birdhouse/config/thredds/service-config.json.template b/birdhouse/config/thredds/service-config.json.template index bc93cbab4..0b293536e 100644 --- a/birdhouse/config/thredds/service-config.json.template +++ b/birdhouse/config/thredds/service-config.json.template @@ -1,7 +1,7 @@ { "name": "thredds", "url": "https://${PAVICS_FQDN_PUBLIC}/thredds", - "type": ["thredds"], + "type": ["data"], "documentation": "https://www.unidata.ucar.edu/software/tds/", "description": "A server that provides catalog, metadata, and data access services for scientific data." } diff --git a/tests/service-config-schema.json b/tests/service-config-schema.json index 40045936c..68dcc95ec 100644 --- a/tests/service-config-schema.json +++ b/tests/service-config-schema.json @@ -21,7 +21,8 @@ "type": "array", "minItems": 1, "items": { - "type": "string" + "type": "string", + "pattern": "^catalog|data|jupyterhub|other|service-(wps|wms|wfs|wcs|ogcapi)$" } }, "documentation": { From 4f80dc1cc9a6ef249869f54dae88b54cb5b54064 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Thu, 1 Jun 2023 16:31:53 -0400 Subject: [PATCH 05/39] suggested clarification edits for running autodeploy tests --- birdhouse/components/README.rst | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index 701ee60ab..841696d42 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -142,15 +142,15 @@ There are 2 tests that need to be performed: * Can autodeploy deploy the PR from ``master`` branch, the stable reference point - * This could fail if some changes, in the PR, are compatible with autodeploy, example ``./pavics-compose.sh`` calls some binaries that do not exist in the autodeploy docker image. + * This could fail if some changes in the PR, are incompatible with autodeploy. For example: ``./pavics-compose.sh`` calls some binaries that do not exist in the autodeploy docker image. -* Can autodeploy be triggered again successfully, after PR is live +* Can autodeploy be triggered again successfully, after the PR is live - * This could fail if PR renamed some files and forgot to add to ``.gitignore`` the old file names. Then old file names will appears as new uncommitted files and autodeploy will halt because it expects a clean working directory. + * This could fail if the PR renamed some files and forgot to add the old file names to a ``.gitignore`` file. Then old file names will appear as new uncommitted files and autodeploy will halt because it expects a clean working directory. Here is a sample setup to test autodeploy: -* Have 2 checkout directories. One is for starting the stack using ``./pavics-compose.sh``, the other one is to push new bogus change to trigger the autodeploy. +* Have 2 checkout directories. One is for starting the stack using ``./pavics-compose.sh``, the other one is to push new bogus changes to trigger the autodeploy mechanism. .. code-block:: shell @@ -160,7 +160,7 @@ Here is a sample setup to test autodeploy: # this one for triggering autodeploy git clone git@github.com:bird-house/birdhouse-deploy.git birdhouse-deploy-trigger -* Set ``AUTODEPLOY_PLATFORM_FREQUENCY`` in ``env.local`` to a very frequent value so you do not wait too long for autodeploy to trigger. +* Set ``AUTODEPLOY_PLATFORM_FREQUENCY`` in ``env.local`` to a very frequent value so you do not have to wait too long for autodeploy to trigger. .. code-block:: shell @@ -176,7 +176,7 @@ Here is a sample setup to test autodeploy: # recreate scheduler container for new AUTODEPLOY_PLATFORM_FREQUENCY to be effective ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d -* Create a ``test`` branch so you can add bogus commits without affecting your real PR. Setup your main checkout to track that test branch so it new changes to trigger. +* Create a ``test`` branch so you can add bogus commits without affecting your real PR. Set up your main checkout (birdhouse-deploy) to track that test branch so it will detect new changes on the test branch and trigger the autodeploy. .. code-block:: shell @@ -206,12 +206,15 @@ Here is a sample setup to test autodeploy: git reset --hard YOUR_PR_BRANCH git push + # now that the remote "test" branch differs from the local "test" branch in the birdhouse-deploy repo, + # the autodeploy mechanism will detect that the remote branch has changed and attempt to update the local branch + # follow logs, check for errors tail -f /var/log/PAVICS/autodeploy.log # do spot checks, run Jenkins on your deployment if needed -* Test scenario 2, from your PR and later +* Test scenario 2, from your PR to later changes .. code-block:: shell @@ -220,9 +223,13 @@ Here is a sample setup to test autodeploy: # add any bogus commit to trigger autodeploy again echo >> README.rst - git ci -am "trigger autodeploy" + git add README.rst + git commit -m "trigger autodeploy" git push + # now that the remote "test" branch differs from the local "test" branch in the birdhouse-deploy repo, + # the autodeploy mechanism will detect that the remote branch has changed and attempt to update the local branch + # follow logs, check for errors tail -f /var/log/PAVICS/autodeploy.log @@ -243,14 +250,16 @@ Here is a sample setup to test autodeploy: # go to YOUR_PR_BRANCH so we can delete the test branch git checkout YOUR_PR_BRANCH - git push origin --delete test git branch -D test # edit env.local and change AUTODEPLOY_PLATFORM_FREQUENCY to something less frequent to save your cpu + # do not remove the scheduler component from the stack yet or the next command will fail # recreate scheduler container for new AUTODEPLOY_PLATFORM_FREQUENCY to be effective ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d + # optionally edit env.local to remove the scheduler component from the stack + Monitoring ========== From 141ba560a611e941bdb5162f1eef3065557669cd Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Fri, 2 Jun 2023 10:05:39 -0400 Subject: [PATCH 06/39] add additional services types to show all possible service types --- birdhouse/components/weaver/service-config.json.template | 2 +- birdhouse/config/geoserver/service-config.json.template | 2 +- birdhouse/config/thredds/service-config.json.template | 2 +- tests/service-config-schema.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/birdhouse/components/weaver/service-config.json.template b/birdhouse/components/weaver/service-config.json.template index d49ce527d..9d8929c86 100644 --- a/birdhouse/components/weaver/service-config.json.template +++ b/birdhouse/components/weaver/service-config.json.template @@ -1,7 +1,7 @@ { "name": "weaver", "url": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}", - "type": ["service-ogcapi"], + "type": ["service-ogcapi_processes"], "documentation": "https://pavics-weaver.readthedocs.io", "description": "An OGC-API flavored Execution Management Service" } diff --git a/birdhouse/config/geoserver/service-config.json.template b/birdhouse/config/geoserver/service-config.json.template index d32203684..d9ba2d5f9 100644 --- a/birdhouse/config/geoserver/service-config.json.template +++ b/birdhouse/config/geoserver/service-config.json.template @@ -1,7 +1,7 @@ { "name": "geoserver", "url": "https://${PAVICS_FQDN_PUBLIC}/geoserver", - "type": ["data", "service-wms"], + "type": ["data", "service-wms", "service-wfs", "service-wps"], "documentation": "https://docs.geoserver.org/", "description": "GeoServer is a server that allows users to view and edit geospatial data." } diff --git a/birdhouse/config/thredds/service-config.json.template b/birdhouse/config/thredds/service-config.json.template index 0b293536e..5e2ca9785 100644 --- a/birdhouse/config/thredds/service-config.json.template +++ b/birdhouse/config/thredds/service-config.json.template @@ -1,7 +1,7 @@ { "name": "thredds", "url": "https://${PAVICS_FQDN_PUBLIC}/thredds", - "type": ["data"], + "type": ["data", "catalog"], "documentation": "https://www.unidata.ucar.edu/software/tds/", "description": "A server that provides catalog, metadata, and data access services for scientific data." } diff --git a/tests/service-config-schema.json b/tests/service-config-schema.json index 68dcc95ec..833362fea 100644 --- a/tests/service-config-schema.json +++ b/tests/service-config-schema.json @@ -22,7 +22,7 @@ "minItems": 1, "items": { "type": "string", - "pattern": "^catalog|data|jupyterhub|other|service-(wps|wms|wfs|wcs|ogcapi)$" + "pattern": "^catalog|data|jupyterhub|other|service-(wps|wms|wfs|wcs|ogcapi_processes)$" } }, "documentation": { From 636985b5de8f0a686ad52fea7ab3b77d8b04409d Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Fri, 2 Jun 2023 10:20:54 -0400 Subject: [PATCH 07/39] fix cowbird config to show service types not names --- CHANGES.md | 4 +++- .../cowbird/config/cowbird/config.yml.template | 9 ++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f021dca3d..705c945cd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,9 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Fixes + +- Fix Cowbird's `sync_permissions` config which used invalid Magpie service types. [1.26.2](https://github.com/bird-house/birdhouse-deploy/tree/1.26.2) (2023-05-25) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index 2066617ed..9e32de893 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -41,8 +41,8 @@ sync_permissions: # For more info on the services available in Magpie : # https://pavics-magpie.readthedocs.io/en/latest/services.html#available-services # https://pavics-magpie.readthedocs.io/en/latest/autoapi/magpie/services/index.html - services: # Contains the different resources that can be synchronized, ordered by service - thredds: # Service name, which should also exist in Magpie + services: # Contains the different resources that can be synchronized, ordered by service type + thredds: # Service type, which should also exist in Magpie # Resource key (ex.: thredds_workspace): Custom name to represent a resource path. # # Example of resource that uses variables and a `MULTI_TOKEN`. @@ -107,7 +107,7 @@ sync_permissions: - "geoserver_workspace : createStoredQuery <-> thredds_workspace : write" weaver_outputs: services: - weaver: + api: process_description: - name: weaver type: service @@ -153,8 +153,7 @@ sync_permissions: type: route - name: "{outputID}" type: route - # see 'optional-components/secure-data-proxy' for more details on protected WPS-outputs - wps_outputs: + # see 'optional-components/secure-data-proxy' for more details on protected WPS-outputs # /wpsoutputs/weaver/{public|}/{job-id} weaver_wps_outputs: - name: secure-data-proxy From d58d79a1d35a49322e4cd4a732cf6867854e6d32 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Mon, 5 Jun 2023 11:45:18 -0400 Subject: [PATCH 08/39] convert version endpoint to display version and release time as json string --- .../config/proxy/conf.d/all-services.include.template | 2 +- birdhouse/pavics-compose.sh | 2 +- birdhouse/scripts/get-version-json.include.sh | 11 +++++++++++ birdhouse/scripts/get-version.include.sh | 9 --------- 4 files changed, 13 insertions(+), 11 deletions(-) create mode 100755 birdhouse/scripts/get-version-json.include.sh delete mode 100755 birdhouse/scripts/get-version.include.sh diff --git a/birdhouse/config/proxy/conf.d/all-services.include.template b/birdhouse/config/proxy/conf.d/all-services.include.template index 0f80f9d6a..e1c9346a8 100644 --- a/birdhouse/config/proxy/conf.d/all-services.include.template +++ b/birdhouse/config/proxy/conf.d/all-services.include.template @@ -14,7 +14,7 @@ location /version { default_type text/plain; - return 200 '${BIRDHOUSE_VERSION}'; + return 200 '${BIRDHOUSE_VERSION_JSON}'; } location /doc { diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index 34de012a9..8e8644b4a 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -50,7 +50,7 @@ read_configs # this sets ALL_CONF_DIRS . ./scripts/get-components-json.include.sh . ./scripts/get-services-json.include.sh -. ./scripts/get-version.include.sh +. ./scripts/get-version-json.include.sh for i in ${VARS} do diff --git a/birdhouse/scripts/get-version-json.include.sh b/birdhouse/scripts/get-version-json.include.sh new file mode 100755 index 000000000..b6844815c --- /dev/null +++ b/birdhouse/scripts/get-version-json.include.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# get the current version of this deployment +RELEASE_FILE="$(dirname "${COMPOSE_DIR}")/RELEASE.txt" + +if [ -f "${RELEASE_FILE}" ]; then + BIRDHOUSE_VERSION="$(head -1 "$RELEASE_FILE" | cut -d" " -f 1)" + BIRDHOUSE_RELEASE_TIME="$(head -1 "$RELEASE_FILE" | cut -d" " -f 2)" + BIRDHOUSE_VERSION_JSON="{\"version\": \"${BIRDHOUSE_VERSION}\", \"release_time\": \"${BIRDHOUSE_RELEASE_TIME}\"}" + export BIRDHOUSE_VERSION_JSON +fi diff --git a/birdhouse/scripts/get-version.include.sh b/birdhouse/scripts/get-version.include.sh deleted file mode 100755 index 72d0a1de1..000000000 --- a/birdhouse/scripts/get-version.include.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -# get the current version of this deployment -RELEASE_FILE="$(dirname "${COMPOSE_DIR}")/RELEASE.txt" - -if [ -f "${RELEASE_FILE}" ]; then - BIRDHOUSE_VERSION="$(head -1 "$RELEASE_FILE" | cut -d" " -f 1)" - export BIRDHOUSE_VERSION -fi From c39a389bf67b6b8b5238ed45f64dd1a14bfc63ba Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Mon, 5 Jun 2023 13:23:14 -0400 Subject: [PATCH 09/39] ensure the proxy variable is replaced in the template --- birdhouse/config/proxy/default.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdhouse/config/proxy/default.env b/birdhouse/config/proxy/default.env index 7a74a2430..65e21656e 100644 --- a/birdhouse/config/proxy/default.env +++ b/birdhouse/config/proxy/default.env @@ -22,7 +22,7 @@ export VARS=" $VARS \$BIRDHOUSE_DEPLOY_COMPONENTS_JSON \$BIRDHOUSE_DEPLOY_SERVICES_JSON - \$BIRDHOUSE_VERSION + \$BIRDHOUSE_VERSION_JSON " export OPTIONAL_VARS=" From 776744017458cb3d68af74702f7621b8fd12c381 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Mon, 5 Jun 2023 13:24:40 -0400 Subject: [PATCH 10/39] proxy endpoint type is json --- birdhouse/config/proxy/conf.d/all-services.include.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdhouse/config/proxy/conf.d/all-services.include.template b/birdhouse/config/proxy/conf.d/all-services.include.template index e1c9346a8..48383e29f 100644 --- a/birdhouse/config/proxy/conf.d/all-services.include.template +++ b/birdhouse/config/proxy/conf.d/all-services.include.template @@ -13,7 +13,7 @@ } location /version { - default_type text/plain; + default_type application/json; return 200 '${BIRDHOUSE_VERSION_JSON}'; } From 34024f869e97f531d29e5addc3db9a6859bedcbc Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Fri, 9 Jun 2023 14:51:31 -0400 Subject: [PATCH 11/39] update schema to support service links --- birdhouse/README.rst | 5 +++ .../weaver/service-config.json.template | 32 +++++++++++++--- .../config/finch/service-config.json.template | 27 ++++++++++--- .../flyingpigeon/service-config.json.template | 27 ++++++++++--- .../geoserver/service-config.json.template | 26 ++++++++++--- .../hummingbird/service-config.json.template | 27 ++++++++++--- .../jupyterhub/service-config.json.template | 22 ++++++++--- .../config/raven/service-config.json.template | 27 ++++++++++--- .../thredds/service-config.json.template | 28 +++++++++++--- tests/requirements.txt | 1 + tests/service-config-schema.json | 38 ------------------- tests/test_deployment.py | 21 ++++++++-- 12 files changed, 199 insertions(+), 82 deletions(-) delete mode 100644 tests/service-config-schema.json diff --git a/birdhouse/README.rst b/birdhouse/README.rst index 7a29e6c4d..f513ad85c 100644 --- a/birdhouse/README.rst +++ b/birdhouse/README.rst @@ -262,6 +262,11 @@ To run the tests: python3 -m pip install -r tests/requirements.txt pytest tests/ +Some tests require internet access (to access JSON schemas used to validate +JSON structure). If you need to run tests offline, you can skip the tests that +require internet access by setting the `DACCS_SKIP_ONLINE_TESTS` environment +variable to a non-empty value. + Tagging policy -------------- diff --git a/birdhouse/components/weaver/service-config.json.template b/birdhouse/components/weaver/service-config.json.template index 9d8929c86..196a3b6ec 100644 --- a/birdhouse/components/weaver/service-config.json.template +++ b/birdhouse/components/weaver/service-config.json.template @@ -1,7 +1,29 @@ { - "name": "weaver", - "url": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}", - "type": ["service-ogcapi_processes"], - "documentation": "https://pavics-weaver.readthedocs.io", - "description": "An OGC-API flavored Execution Management Service" + "name": "weaver", + "keywords": [ + "service-ogcapi_processes" + ], + "description": "An OGC-API flavored Execution Management Service", + "links": [ + { + "rel": "service", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://pavics-weaver.readthedocs.io/" + }, + { + "rel": "service-desc", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" + }, + { + "rel": "conformance", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/conformance/" + } + ] } diff --git a/birdhouse/config/finch/service-config.json.template b/birdhouse/config/finch/service-config.json.template index a9e3f9a8b..586cbaab1 100644 --- a/birdhouse/config/finch/service-config.json.template +++ b/birdhouse/config/finch/service-config.json.template @@ -1,7 +1,24 @@ { - "name": "finch", - "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch", - "type": ["service-wps"], - "documentation": "https://finch.readthedocs.io", - "description": "A Web Processing Service for Climate Indicators." + "name": "finch", + "keywords": [ + "service-wps" + ], + "description": "A Web Processing Service for Climate Indicators.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://finch.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" + } + ] } diff --git a/birdhouse/config/flyingpigeon/service-config.json.template b/birdhouse/config/flyingpigeon/service-config.json.template index 0ab2c3d99..17b1d22a8 100644 --- a/birdhouse/config/flyingpigeon/service-config.json.template +++ b/birdhouse/config/flyingpigeon/service-config.json.template @@ -1,7 +1,24 @@ { - "name": "flyingpigeon", - "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon", - "type": ["service-wps"], - "documentation": "https://flyingpigeon.readthedocs.io", - "description": "WPS for climate model data, indices and extreme events." + "name": "flyingpigeon", + "keywords": [ + "service-wps" + ], + "description": "WPS for climate model data, indices and extreme events.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://flyingpigeon.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" + } + ] } diff --git a/birdhouse/config/geoserver/service-config.json.template b/birdhouse/config/geoserver/service-config.json.template index d9ba2d5f9..9049e2dd0 100644 --- a/birdhouse/config/geoserver/service-config.json.template +++ b/birdhouse/config/geoserver/service-config.json.template @@ -1,7 +1,23 @@ { - "name": "geoserver", - "url": "https://${PAVICS_FQDN_PUBLIC}/geoserver", - "type": ["data", "service-wms", "service-wfs", "service-wps"], - "documentation": "https://docs.geoserver.org/", - "description": "GeoServer is a server that allows users to view and edit geospatial data." + "name": "geoserver", + "keywords": [ + "data", + "service-wms", + "service-wfs", + "service-wps" + ], + "documentation": "https://docs.geoserver.org/", + "description": "GeoServer is a server that allows users to view and edit geospatial data.", + "links": [ + { + "rel": "service", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/geoserver/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://docs.geoserver.org/" + } + ] } diff --git a/birdhouse/config/hummingbird/service-config.json.template b/birdhouse/config/hummingbird/service-config.json.template index 6d44d6b4c..8ad115194 100644 --- a/birdhouse/config/hummingbird/service-config.json.template +++ b/birdhouse/config/hummingbird/service-config.json.template @@ -1,7 +1,24 @@ { - "name": "hummingbird", - "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird", - "type": ["service-wps"], - "documentation": "https://birdhouse-hummingbird.readthedocs.io", - "description": "A WPS for metadata compliance checks used in the climate science community." + "name": "hummingbird", + "keywords": [ + "service-wps" + ], + "description": "A WPS for metadata compliance checks used in the climate science community.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://birdhouse-hummingbird.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" + } + ] } diff --git a/birdhouse/config/jupyterhub/service-config.json.template b/birdhouse/config/jupyterhub/service-config.json.template index ee4c3b6eb..747103e3c 100644 --- a/birdhouse/config/jupyterhub/service-config.json.template +++ b/birdhouse/config/jupyterhub/service-config.json.template @@ -1,7 +1,19 @@ { - "name": "jupyterhub", - "url": "https://${PAVICS_FQDN_PUBLIC}/jupyter", - "type": ["jupyterhub"], - "documentation": "https://jupyter.org/hub", - "description": "An interactive development environment use to create and share documents that contain live code." + "name": "jupyterhub", + "keywords": [ + "jupyterhub" + ], + "description": "An interactive development environment use to create and share documents that contain live code.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://${PAVICS_FQDN_PUBLIC}/jupyter" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://jupyter.org/hub" + } + ] } diff --git a/birdhouse/config/raven/service-config.json.template b/birdhouse/config/raven/service-config.json.template index 0b119a8ae..7e586edf5 100644 --- a/birdhouse/config/raven/service-config.json.template +++ b/birdhouse/config/raven/service-config.json.template @@ -1,7 +1,24 @@ { - "name": "raven", - "url": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven", - "type": ["service-wps"], - "documentation": "https://pavics-raven.readthedocs.io", - "description": "A WPS providing hydrological modeling and analysis services." + "name": "raven", + "keywords": [ + "service-wps" + ], + "description": "A WPS providing hydrological modeling and analysis services.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://pavics-raven.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" + } + ] } diff --git a/birdhouse/config/thredds/service-config.json.template b/birdhouse/config/thredds/service-config.json.template index 5e2ca9785..01b0b8356 100644 --- a/birdhouse/config/thredds/service-config.json.template +++ b/birdhouse/config/thredds/service-config.json.template @@ -1,7 +1,25 @@ { - "name": "thredds", - "url": "https://${PAVICS_FQDN_PUBLIC}/thredds", - "type": ["data", "catalog"], - "documentation": "https://www.unidata.ucar.edu/software/tds/", - "description": "A server that provides catalog, metadata, and data access services for scientific data." + "name": "thredds", + "keywords": [ + "data", + "catalog" + ], + "description": "A server that provides catalog, metadata, and data access services for scientific data.", + "links": [ + { + "rel": "service", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/thredds/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://www.unidata.ucar.edu/software/tds/" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}/thredds/catalog.xml" + } + ] } diff --git a/tests/requirements.txt b/tests/requirements.txt index 876fb2f39..212d9bfa3 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,3 @@ pytest==7.2.2 jsonschema==4.17.3 +requests==2.31.0 diff --git a/tests/service-config-schema.json b/tests/service-config-schema.json deleted file mode 100644 index 833362fea..000000000 --- a/tests/service-config-schema.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "type": "object", - "required": [ - "url", - "name", - "type", - "documentation", - "description" - ], - "properties": { - "href": { - "type": "string", - "format": "uri", - "pattern": "^https?://" - }, - "name": { - "type": "string", - "minLength": 1 - }, - "type": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "pattern": "^catalog|data|jupyterhub|other|service-(wps|wms|wfs|wcs|ogcapi_processes)$" - } - }, - "documentation": { - "type": "string", - "format": "uri", - "pattern": "^https?://" - }, - "description": { - "type": "string", - "minLength": 1 - } - } -} \ No newline at end of file diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 0d576441f..6539630be 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -5,12 +5,13 @@ from string import Template import jsonschema +import requests COMPONENT_LOCATIONS = ("components", "optional-components", "config") TEMPLATE_SUBSTITUTIONS = { "PAVICS_FQDN_PUBLIC": os.environ.get("PAVICS_FQDN_PUBLIC", "example.com"), "WEAVER_MANAGER_NAME": os.environ.get("WEAVER_MANAGER_NAME", "weaver"), - "TWITCHER_PROTECTED_PATH": os.environ.get("TWITCHER_PROTECTED_PATH", "/twitcher/ows/proxy") + "TWITCHER_PROTECTED_PATH": os.environ.get("TWITCHER_PROTECTED_PATH", "/twitcher/ows/proxy"), } @@ -25,9 +26,13 @@ def component_paths(root_dir): @pytest.fixture(scope="module") -def services_config_schema(request): - with open(os.path.join(os.path.dirname(request.fspath), "service-config-schema.json")) as f: - return json.load(f) +def services_config_schema(): + branch = os.environ.get("DACCS_NODE_REGISTRY_BRANCH", "main") + schema = { + "$ref": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry" + f"/{branch}/node_registry.schema.json#service" + } + return schema class TestDockerCompose: @@ -57,4 +62,12 @@ def test_service_config_valid(self, component_paths, services_config_schema): jsonschema.validate(instance=service_config, schema=services_config_schema) except jsonschema.exceptions.ValidationError as e: invalid_schemas.append(f"{os.path.basename(path)} contains invalid service configuration: {e}") + except jsonschema.exceptions.RefResolutionError as e: + if os.environ.get("DACCS_SKIP_ONLINE_TESTS"): + pytest.skip("This test requires an internet connection, skipping") + else: + pytest.fail( + f"This test requires an internet connection. To skip this test instead set the" + " DACCS_SKIP_ONLINE_TESTS environment variable failing with error: \n{e}" + ) assert not invalid_schemas, "\n".join(invalid_schemas) From 4d0efad247f1d8df9b28da40f4e67744756c943f Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Mon, 12 Jun 2023 09:49:24 -0400 Subject: [PATCH 12/39] serve services, components, and version information from a file --- birdhouse/config/proxy/.gitignore | 1 + .../proxy/conf.d/all-services.include.template | 9 ++++++--- birdhouse/config/proxy/docker-compose-extra.yml | 1 + birdhouse/config/proxy/pre-docker-compose-up | 12 ++++++++++++ birdhouse/scripts/get-services-json.include.sh | 8 +++----- 5 files changed, 23 insertions(+), 8 deletions(-) create mode 100755 birdhouse/config/proxy/pre-docker-compose-up diff --git a/birdhouse/config/proxy/.gitignore b/birdhouse/config/proxy/.gitignore index de4610cf8..c2bb0d35d 100644 --- a/birdhouse/config/proxy/.gitignore +++ b/birdhouse/config/proxy/.gitignore @@ -1,3 +1,4 @@ conf.d/frontend.conf conf.d/all-services.include nginx.conf +static/ diff --git a/birdhouse/config/proxy/conf.d/all-services.include.template b/birdhouse/config/proxy/conf.d/all-services.include.template index 48383e29f..b3222321f 100644 --- a/birdhouse/config/proxy/conf.d/all-services.include.template +++ b/birdhouse/config/proxy/conf.d/all-services.include.template @@ -4,17 +4,20 @@ location /components/ { default_type application/json; - return 200 '${BIRDHOUSE_DEPLOY_COMPONENTS_JSON}'; + root /static; + try_files /components.json =404; } location /services { default_type application/json; - return 200 '${BIRDHOUSE_DEPLOY_SERVICES_JSON}'; + root /static; + try_files /services.json =404; } location /version { default_type application/json; - return 200 '${BIRDHOUSE_VERSION_JSON}'; + root /static; + try_files /version.json =404; } location /doc { diff --git a/birdhouse/config/proxy/docker-compose-extra.yml b/birdhouse/config/proxy/docker-compose-extra.yml index 6509aca6c..f212ae7aa 100644 --- a/birdhouse/config/proxy/docker-compose-extra.yml +++ b/birdhouse/config/proxy/docker-compose-extra.yml @@ -23,6 +23,7 @@ services: - ./config/proxy/conf.d:/etc/nginx/conf.d - ./config/proxy/nginx.conf:/etc/nginx/nginx.conf - ${SSL_CERTIFICATE}:/etc/nginx/cert.pem + - ./config/proxy/static:/static environment: # https://github.com/bird-house/birdhouse-deploy/issues/198 REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt diff --git a/birdhouse/config/proxy/pre-docker-compose-up b/birdhouse/config/proxy/pre-docker-compose-up new file mode 100755 index 000000000..e0900cf8d --- /dev/null +++ b/birdhouse/config/proxy/pre-docker-compose-up @@ -0,0 +1,12 @@ +#!/bin/sh + +# Create JSON files containing the version information, available services, and +# enabled components. These files will be served by the nginx proxy as static files. + +THIS_FILE="$(realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" + +echo "${BIRDHOUSE_VERSION_JSON}" > "${THIS_DIR}/static/version.json" +echo "${BIRDHOUSE_DEPLOY_SERVICES_JSON}" > "${THIS_DIR}/static/services.json" +echo "${BIRDHOUSE_DEPLOY_COMPONENTS_JSON}" > "${THIS_DIR}/static/components.json" + diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh index df3b6a15a..713a20057 100755 --- a/birdhouse/scripts/get-services-json.include.sh +++ b/birdhouse/scripts/get-services-json.include.sh @@ -1,15 +1,13 @@ #!/bin/sh # default value in case of error or missing definitions -export BIRDHOUSE_DEPLOY_SERVICES_JSON='{}' -if [ -z "${ALL_CONF_DIRS}" ]; then - echo "No services in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. SERVICES JSON list will be empty!" - return -fi for adir in ${ALL_CONF_DIRS}; do [ -f "${adir}/service-config.json" ] || continue SERVICES="${SERVICES}$([ -n "${SERVICES}" ] && echo ',') $(cat "${adir}/service-config.json")" done +if [ -z "${SERVICES}" ]; then + echo "No services in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. SERVICES JSON list will be empty!" +fi export BIRDHOUSE_DEPLOY_SERVICES_JSON="{\"services\": [${SERVICES}]}" From a6cafd90fdb69794b223d4a8df924e29fcc1ae8f Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Wed, 14 Jun 2023 09:46:41 -0400 Subject: [PATCH 13/39] pre-create static dir --- birdhouse/config/proxy/pre-docker-compose-up | 2 ++ 1 file changed, 2 insertions(+) diff --git a/birdhouse/config/proxy/pre-docker-compose-up b/birdhouse/config/proxy/pre-docker-compose-up index e0900cf8d..0d5acbe8c 100755 --- a/birdhouse/config/proxy/pre-docker-compose-up +++ b/birdhouse/config/proxy/pre-docker-compose-up @@ -6,6 +6,8 @@ THIS_FILE="$(realpath "$0")" THIS_DIR="$(dirname "$THIS_FILE")" +mkdir -p "${THIS_DIR}/static" + echo "${BIRDHOUSE_VERSION_JSON}" > "${THIS_DIR}/static/version.json" echo "${BIRDHOUSE_DEPLOY_SERVICES_JSON}" > "${THIS_DIR}/static/services.json" echo "${BIRDHOUSE_DEPLOY_COMPONENTS_JSON}" > "${THIS_DIR}/static/components.json" From bc4ff68c664ba738933568a0715576f328d20d6d Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Wed, 14 Jun 2023 09:48:35 -0400 Subject: [PATCH 14/39] remove duplicate documentation link --- birdhouse/config/geoserver/service-config.json.template | 1 - 1 file changed, 1 deletion(-) diff --git a/birdhouse/config/geoserver/service-config.json.template b/birdhouse/config/geoserver/service-config.json.template index 9049e2dd0..c7158c36c 100644 --- a/birdhouse/config/geoserver/service-config.json.template +++ b/birdhouse/config/geoserver/service-config.json.template @@ -6,7 +6,6 @@ "service-wfs", "service-wps" ], - "documentation": "https://docs.geoserver.org/", "description": "GeoServer is a server that allows users to view and edit geospatial data.", "links": [ { From c66c0813082fb93be85775a693eaa875982bf386 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Wed, 14 Jun 2023 11:13:37 -0400 Subject: [PATCH 15/39] fix user permissions on files when checking out new files from git --- birdhouse/components/scheduler/config.yml.template | 1 + birdhouse/components/scheduler/default.env | 5 +++++ birdhouse/components/scheduler/docker-compose-extra.yml | 1 + birdhouse/deployment/deploy.sh | 7 +++++++ birdhouse/env.local.example | 7 ++++++- 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/birdhouse/components/scheduler/config.yml.template b/birdhouse/components/scheduler/config.yml.template index 2f5e75e80..81d863c76 100644 --- a/birdhouse/components/scheduler/config.yml.template +++ b/birdhouse/components/scheduler/config.yml.template @@ -40,6 +40,7 @@ --env COMPOSE_DIR=${COMPOSE_DIR} --env AUTODEPLOY_DEPLOY_KEY_ROOT_DIR=${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR} --env JUPYTERHUB_USER_DATA_DIR=${JUPYTERHUB_USER_DATA_DIR} + --env CODE_OWNERSHIP=${CODE_OWNERSHIP} --env AUTODEPLOY_SILENT=true${AUTODEPLOY_PLATFORM_EXTRA_DOCKER_ARGS} image: 'pavics/docker-compose-git:docker-18.09.7-compose-1.25.1' diff --git a/birdhouse/components/scheduler/default.env b/birdhouse/components/scheduler/default.env index bc6339829..07c684db9 100644 --- a/birdhouse/components/scheduler/default.env +++ b/birdhouse/components/scheduler/default.env @@ -27,6 +27,11 @@ if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then fi fi +# The scheduler runs as the root user so new/updated files will be owned by root after the code is updated, +# owner of the code to AUTODEPLOY_CODE_OWNERSHIP if set. AUTODEPLOY_CODE_OWNERSHIP should contain uids instead of +# usernames since usernames within a docker container will not necessarily line up with those on the host system. +export AUTODEPLOY_CODE_OWNERSHIP="$(id -u):$(id -u)" + # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here OPTIONAL_VARS=" $OPTIONAL_VARS diff --git a/birdhouse/components/scheduler/docker-compose-extra.yml b/birdhouse/components/scheduler/docker-compose-extra.yml index e3415ae29..a7faf401c 100644 --- a/birdhouse/components/scheduler/docker-compose-extra.yml +++ b/birdhouse/components/scheduler/docker-compose-extra.yml @@ -10,6 +10,7 @@ services: environment: COMPOSE_DIR: ${PWD} AUTODEPLOY_DEPLOY_KEY_ROOT_DIR: ${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR} + CODE_OWNERSHIP: ${AUTODEPLOY_CODE_OWNERSHIP} restart: always # vi: tabstop=8 expandtab shiftwidth=2 softtabstop=2 diff --git a/birdhouse/deployment/deploy.sh b/birdhouse/deployment/deploy.sh index 987bf73ff..691909321 100755 --- a/birdhouse/deployment/deploy.sh +++ b/birdhouse/deployment/deploy.sh @@ -156,6 +156,13 @@ for adir in $COMPOSE_DIR $AUTODEPLOY_EXTRA_REPOS; do # pull the current branch, so this deploy script supports any branches, not just master git pull + + # This runs as the root user so new/updated files will be owned by root after the git pull, this sets the + # owner of the code to CODE_OWNERSHIP if set. CODE_OWNERSHIP should contain uids instead of usernames since + # usernames within a docker container will not necessarily line up with those on the host system. + if [ -n "$CODE_OWNERSHIP" ]; then + chown -R "$CODE_OWNERSHIP" . + fi else echo "WARNING: extra repo '$adir' do not exist" fi diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 869453742..3089dbaae 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -169,7 +169,12 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass # Potential usages: other deployment, backup jobs on the same machine # #export AUTODEPLOY_EXTRA_SCHEDULER_JOBS="" -# + +# The scheduler runs as the root user so new/updated files will be owned by root after the code is updated, +# owner of the code to AUTODEPLOY_CODE_OWNERSHIP if set. AUTODEPLOY_CODE_OWNERSHIP should contain uids instead of +# usernames since usernames within a docker container will not necessarily line up with those on the host system. +#export AUTODEPLOY_CODE_OWNERSHIP="$(id -u):$(id -u)" + # Load pre-configured job to auto-renew LetsEncrypt SSL certificate if a # LetsEncrypt SSL certificate has previously been requested. # From e16608c7a231df3050a775887960546ae9dd5c91 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Wed, 14 Jun 2023 11:42:30 -0400 Subject: [PATCH 16/39] ensure that recursive chown happens from top level or repo --- birdhouse/deployment/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdhouse/deployment/deploy.sh b/birdhouse/deployment/deploy.sh index 691909321..7159ae5e3 100755 --- a/birdhouse/deployment/deploy.sh +++ b/birdhouse/deployment/deploy.sh @@ -161,7 +161,7 @@ for adir in $COMPOSE_DIR $AUTODEPLOY_EXTRA_REPOS; do # owner of the code to CODE_OWNERSHIP if set. CODE_OWNERSHIP should contain uids instead of usernames since # usernames within a docker container will not necessarily line up with those on the host system. if [ -n "$CODE_OWNERSHIP" ]; then - chown -R "$CODE_OWNERSHIP" . + chown -R "$CODE_OWNERSHIP" "$(git rev-parse --show-toplevel)" fi else echo "WARNING: extra repo '$adir' do not exist" From a0cc838fb0097671528b0ce64a4c42a0618475e8 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Wed, 14 Jun 2023 12:14:21 -0400 Subject: [PATCH 17/39] hardcode the environment variable so that it remains consistent inside and outside docker containers --- birdhouse/components/scheduler/default.env | 5 ----- birdhouse/env.local.example | 12 ++++++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/birdhouse/components/scheduler/default.env b/birdhouse/components/scheduler/default.env index 07c684db9..bc6339829 100644 --- a/birdhouse/components/scheduler/default.env +++ b/birdhouse/components/scheduler/default.env @@ -27,11 +27,6 @@ if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then fi fi -# The scheduler runs as the root user so new/updated files will be owned by root after the code is updated, -# owner of the code to AUTODEPLOY_CODE_OWNERSHIP if set. AUTODEPLOY_CODE_OWNERSHIP should contain uids instead of -# usernames since usernames within a docker container will not necessarily line up with those on the host system. -export AUTODEPLOY_CODE_OWNERSHIP="$(id -u):$(id -u)" - # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here OPTIONAL_VARS=" $OPTIONAL_VARS diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 3089dbaae..964e3e3ca 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -170,10 +170,14 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass # #export AUTODEPLOY_EXTRA_SCHEDULER_JOBS="" -# The scheduler runs as the root user so new/updated files will be owned by root after the code is updated, -# owner of the code to AUTODEPLOY_CODE_OWNERSHIP if set. AUTODEPLOY_CODE_OWNERSHIP should contain uids instead of -# usernames since usernames within a docker container will not necessarily line up with those on the host system. -#export AUTODEPLOY_CODE_OWNERSHIP="$(id -u):$(id -u)" +# The scheduler runs as the root user so new/updated files will be owned by root after the code is updated. +# If AUTODEPLOY_CODE_OWNERSHIP if set, the ownership of all files and directories in this repo will be set to that user +# after each autodeploy update. +# AUTODEPLOY_CODE_OWNERSHIP should contain uids instead of usernames since usernames within a docker container will +# not necessarily line up with those on the host system. +# AUTODEPLOY_CODE_OWNERSHIP should be set to a constant value (eg. "1000" not "$(id -u)") since this will be evaluated +# within the autodeploy docker container as the root user. +#export AUTODEPLOY_CODE_OWNERSHIP="1000:1000" # Load pre-configured job to auto-renew LetsEncrypt SSL certificate if a # LetsEncrypt SSL certificate has previously been requested. From 613cfd3dca4e6c93da159bef1443d2fff473162c Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Wed, 14 Jun 2023 12:20:04 -0400 Subject: [PATCH 18/39] update CHANGES.md --- CHANGES.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 152e51067..e472d51fb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,16 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Fixes +- Autodeploy: optionally fix file permissions + + The autodeploy mechanism creates new files owned by root. If this is not desired then users have to manually + update the file ownership after each autodeployment. This adds an option to change the ownership of all files + to a specific user after each autodeployment. + + For example, if the code in this repo is currently owned by a user named `birduser` with uid 1002, then by + setting `export AUTODEPLOY_CODE_OWNERSHIP="1002:1002"` in `env.local`, all files and folders in this repo will + continue to be owned by `birduser` after each autodeployment. [1.26.4](https://github.com/bird-house/birdhouse-deploy/tree/1.26.4) (2023-06-06) ------------------------------------------------------------------------------------------------------------------ From faca3143bdbaa1b82b3bd9f50d249f0f9e9c4ca1 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Wed, 14 Jun 2023 15:46:19 -0400 Subject: [PATCH 19/39] autodeploy testing doc: remove trailing white space on empty line --- birdhouse/components/README.rst | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index 841696d42..73f3a4dce 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -156,7 +156,7 @@ Here is a sample setup to test autodeploy: # this one for running pavics-compose.sh git clone git@github.com:bird-house/birdhouse-deploy.git birdhouse-deploy - + # this one for triggering autodeploy git clone git@github.com:bird-house/birdhouse-deploy.git birdhouse-deploy-trigger @@ -166,33 +166,33 @@ Here is a sample setup to test autodeploy: # go to the main checkout cd birdhouse-deploy/birdhouse - + # ensure the scheduler component is enabled, otherwise autodeploy will not work echo 'export EXTRA_CONF_DIRS="$EXTRA_CONF_DIRS ./components/scheduler" >> env.local - + # set AUTODEPLOY_PLATFORM_FREQUENCY echo 'export AUTODEPLOY_PLATFORM_FREQUENCY="@every 5m"' >> env.local - + # recreate scheduler container for new AUTODEPLOY_PLATFORM_FREQUENCY to be effective ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d - + * Create a ``test`` branch so you can add bogus commits without affecting your real PR. Set up your main checkout (birdhouse-deploy) to track that test branch so it will detect new changes on the test branch and trigger the autodeploy. .. code-block:: shell # go to the main checkout cd birdhouse-deploy/birdhouse - + # initially create the test branch from master git checkout master git pull git checkout -b test git push -u test - + # ensure your runnings code is at "master" and is working correctly # if you do not have a working baseline, you will not know if the breakage is due to autodeploy or your code ./pavics-compose.sh up -d - + * Test scenario 1, from ``master`` to your PR .. code-block:: shell @@ -205,15 +205,15 @@ Here is a sample setup to test autodeploy: git checkout test git reset --hard YOUR_PR_BRANCH git push - + # now that the remote "test" branch differs from the local "test" branch in the birdhouse-deploy repo, # the autodeploy mechanism will detect that the remote branch has changed and attempt to update the local branch - + # follow logs, check for errors tail -f /var/log/PAVICS/autodeploy.log - + # do spot checks, run Jenkins on your deployment if needed - + * Test scenario 2, from your PR to later changes .. code-block:: shell @@ -251,15 +251,15 @@ Here is a sample setup to test autodeploy: # go to YOUR_PR_BRANCH so we can delete the test branch git checkout YOUR_PR_BRANCH git branch -D test - + # edit env.local and change AUTODEPLOY_PLATFORM_FREQUENCY to something less frequent to save your cpu # do not remove the scheduler component from the stack yet or the next command will fail # recreate scheduler container for new AUTODEPLOY_PLATFORM_FREQUENCY to be effective ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d - + # optionally edit env.local to remove the scheduler component from the stack - + Monitoring ========== From 28e66f5b32f2af208feb3eb5b74d6b34dc4fa496 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Wed, 14 Jun 2023 15:55:53 -0400 Subject: [PATCH 20/39] CHANGES: document autodeploy test procedure --- CHANGES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 152e51067..2eec50b54 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,9 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Changes +- Autodeploy: document test procedure + [1.26.4](https://github.com/bird-house/birdhouse-deploy/tree/1.26.4) (2023-06-06) ------------------------------------------------------------------------------------------------------------------ From 8625d4af9cee065bb173f51a1cff2e174df4e8a3 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Wed, 14 Jun 2023 15:58:52 -0400 Subject: [PATCH 21/39] dev tools: add environment-dev.yml to easily install all the dev tools required --- CHANGES.md | 2 ++ environment-dev.yml | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 environment-dev.yml diff --git a/CHANGES.md b/CHANGES.md index 2eec50b54..34d1f1a81 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,8 @@ ## Changes - Autodeploy: document test procedure +- Dev environment: add Conda `environment-dev.yml` to easily install all the dev tools required + [1.26.4](https://github.com/bird-house/birdhouse-deploy/tree/1.26.4) (2023-06-06) ------------------------------------------------------------------------------------------------------------------ diff --git a/environment-dev.yml b/environment-dev.yml new file mode 100644 index 000000000..aee5f7798 --- /dev/null +++ b/environment-dev.yml @@ -0,0 +1,10 @@ +# conda env create -f environment-dev.yml +name: birdhouse-deploy +channels: + - conda-forge # for bump2version +dependencies: + - make + - bump2version + - pytest + - jsonschema + - requests From 84473fed6f6bfcd11e42b7d99517875c13bfff28 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Thu, 15 Jun 2023 09:05:41 -0400 Subject: [PATCH 22/39] fix component script for all versions of grep --- CHANGES.md | 11 ++++++++++- birdhouse/scripts/get-components-json.include.sh | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 152e51067..cbc95953d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,16 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Fixes +- `components/` endpoint displays intended information after auto-deploy + + Previously, the script that generates the content for the `components/` endpoint + was using a feature of `grep` that is not supported by all versions of `grep`. + This meant that this script running in the auto-deployment docker container was + not able to properly parse the running components using `grep`. + This fixes the issue by making the script compliant with all versions of `grep`. + + Resolves https://github.com/bird-house/birdhouse-deploy/issues/342 [1.26.4](https://github.com/bird-house/birdhouse-deploy/tree/1.26.4) (2023-06-06) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/scripts/get-components-json.include.sh b/birdhouse/scripts/get-components-json.include.sh index b738128b8..81aedd895 100755 --- a/birdhouse/scripts/get-components-json.include.sh +++ b/birdhouse/scripts/get-components-json.include.sh @@ -37,6 +37,8 @@ cd "${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}" || true # ignore error for now, empty BIRDHOUSE_DEPLOY_COMPONENTS_LIST_KNOWN="$( \ ls -d1 ./*components/*/ ./config/*/ 2>/dev/null \ | sed -E "s|\./(.*)/|\1|" \ + | sed -E '/^[[:space:]]*$/d' \ + | sed -E 's/^|[[:space:]]+/ -e /' \ )" if [ -z "${BIRDHOUSE_DEPLOY_COMPONENTS_LIST_KNOWN}" ]; then echo "[WARNING]" \ @@ -56,7 +58,7 @@ BIRDHOUSE_DEPLOY_COMPONENTS_LIST_ACTIVE=$( \ BIRDHOUSE_DEPLOY_COMPONENTS_BASE="bird-house/birdhouse-deploy:" BIRDHOUSE_DEPLOY_COMPONENTS_LIST=$( \ echo "${BIRDHOUSE_DEPLOY_COMPONENTS_LIST_ACTIVE}" \ - | grep "${BIRDHOUSE_DEPLOY_COMPONENTS_LIST_KNOWN}" \ + | grep ${BIRDHOUSE_DEPLOY_COMPONENTS_LIST_KNOWN} \ | sed -E 's|^\s*([A-Za-z0-0./_-]+)\s*$|"\1",|g' \ | sed -E "s|^\"\./(.*)\"|\"${BIRDHOUSE_DEPLOY_COMPONENTS_BASE}\\1\"|g" \ | sed '/^\n*$/d' \ From 71286f0a1759bd29f3d1e1480aef288b8b44de52 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Thu, 15 Jun 2023 15:20:22 -0400 Subject: [PATCH 23/39] review suggestions --- birdhouse/README.rst | 3 +-- .../config/proxy/conf.d/all-services.include.template | 2 +- birdhouse/config/thredds/service-config.json.template | 2 +- birdhouse/scripts/get-services-json.include.sh | 2 +- tests/test_deployment.py | 9 +-------- 5 files changed, 5 insertions(+), 13 deletions(-) diff --git a/birdhouse/README.rst b/birdhouse/README.rst index f513ad85c..e3e3c57c5 100644 --- a/birdhouse/README.rst +++ b/birdhouse/README.rst @@ -264,8 +264,7 @@ To run the tests: Some tests require internet access (to access JSON schemas used to validate JSON structure). If you need to run tests offline, you can skip the tests that -require internet access by setting the `DACCS_SKIP_ONLINE_TESTS` environment -variable to a non-empty value. +require internet access by using the `-k 'not online'` pytest option. Tagging policy diff --git a/birdhouse/config/proxy/conf.d/all-services.include.template b/birdhouse/config/proxy/conf.d/all-services.include.template index b3222321f..5704bb12b 100644 --- a/birdhouse/config/proxy/conf.d/all-services.include.template +++ b/birdhouse/config/proxy/conf.d/all-services.include.template @@ -2,7 +2,7 @@ ${PROXY_ROOT_LOCATION} } - location /components/ { + location /components { default_type application/json; root /static; try_files /components.json =404; diff --git a/birdhouse/config/thredds/service-config.json.template b/birdhouse/config/thredds/service-config.json.template index 01b0b8356..75cb55622 100644 --- a/birdhouse/config/thredds/service-config.json.template +++ b/birdhouse/config/thredds/service-config.json.template @@ -8,7 +8,7 @@ "links": [ { "rel": "service", - "type": "application/json", + "type": "text/html", "href": "https://${PAVICS_FQDN_PUBLIC}/thredds/" }, { diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh index 713a20057..fd5a9d257 100755 --- a/birdhouse/scripts/get-services-json.include.sh +++ b/birdhouse/scripts/get-services-json.include.sh @@ -8,6 +8,6 @@ for adir in ${ALL_CONF_DIRS}; do done if [ -z "${SERVICES}" ]; then - echo "No services in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. SERVICES JSON list will be empty!" + echo "${YELLOW}Warning: ${NORMAL}No services in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. SERVICES JSON list will be empty!" fi export BIRDHOUSE_DEPLOY_SERVICES_JSON="{\"services\": [${SERVICES}]}" diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 6539630be..1a034adb9 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -50,6 +50,7 @@ def test_service_config_name_same_as_dirname(self, component_paths): invalid_names.append((config_name, path_name)) assert not invalid_names, "service names in service-config.json.template should match the directory name" + @pytest.mark.online def test_service_config_valid(self, component_paths, services_config_schema): invalid_schemas = [] @@ -62,12 +63,4 @@ def test_service_config_valid(self, component_paths, services_config_schema): jsonschema.validate(instance=service_config, schema=services_config_schema) except jsonschema.exceptions.ValidationError as e: invalid_schemas.append(f"{os.path.basename(path)} contains invalid service configuration: {e}") - except jsonschema.exceptions.RefResolutionError as e: - if os.environ.get("DACCS_SKIP_ONLINE_TESTS"): - pytest.skip("This test requires an internet connection, skipping") - else: - pytest.fail( - f"This test requires an internet connection. To skip this test instead set the" - " DACCS_SKIP_ONLINE_TESTS environment variable failing with error: \n{e}" - ) assert not invalid_schemas, "\n".join(invalid_schemas) From 27c659c4805ff291988fd42e8113bb1a63beea9d Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Thu, 15 Jun 2023 15:24:21 -0400 Subject: [PATCH 24/39] add schema reference to json template --- birdhouse/components/weaver/service-config.json.template | 1 + birdhouse/config/finch/service-config.json.template | 1 + birdhouse/config/flyingpigeon/service-config.json.template | 1 + birdhouse/config/geoserver/service-config.json.template | 1 + birdhouse/config/hummingbird/service-config.json.template | 1 + birdhouse/config/jupyterhub/service-config.json.template | 1 + birdhouse/config/raven/service-config.json.template | 1 + birdhouse/config/thredds/service-config.json.template | 1 + 8 files changed, 8 insertions(+) diff --git a/birdhouse/components/weaver/service-config.json.template b/birdhouse/components/weaver/service-config.json.template index 196a3b6ec..ff0a7f4bc 100644 --- a/birdhouse/components/weaver/service-config.json.template +++ b/birdhouse/components/weaver/service-config.json.template @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", "name": "weaver", "keywords": [ "service-ogcapi_processes" diff --git a/birdhouse/config/finch/service-config.json.template b/birdhouse/config/finch/service-config.json.template index 586cbaab1..bdefd95d7 100644 --- a/birdhouse/config/finch/service-config.json.template +++ b/birdhouse/config/finch/service-config.json.template @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", "name": "finch", "keywords": [ "service-wps" diff --git a/birdhouse/config/flyingpigeon/service-config.json.template b/birdhouse/config/flyingpigeon/service-config.json.template index 17b1d22a8..b576cd6e6 100644 --- a/birdhouse/config/flyingpigeon/service-config.json.template +++ b/birdhouse/config/flyingpigeon/service-config.json.template @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", "name": "flyingpigeon", "keywords": [ "service-wps" diff --git a/birdhouse/config/geoserver/service-config.json.template b/birdhouse/config/geoserver/service-config.json.template index c7158c36c..3cd7670c8 100644 --- a/birdhouse/config/geoserver/service-config.json.template +++ b/birdhouse/config/geoserver/service-config.json.template @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", "name": "geoserver", "keywords": [ "data", diff --git a/birdhouse/config/hummingbird/service-config.json.template b/birdhouse/config/hummingbird/service-config.json.template index 8ad115194..d5d8517c3 100644 --- a/birdhouse/config/hummingbird/service-config.json.template +++ b/birdhouse/config/hummingbird/service-config.json.template @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", "name": "hummingbird", "keywords": [ "service-wps" diff --git a/birdhouse/config/jupyterhub/service-config.json.template b/birdhouse/config/jupyterhub/service-config.json.template index 747103e3c..b09736cff 100644 --- a/birdhouse/config/jupyterhub/service-config.json.template +++ b/birdhouse/config/jupyterhub/service-config.json.template @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", "name": "jupyterhub", "keywords": [ "jupyterhub" diff --git a/birdhouse/config/raven/service-config.json.template b/birdhouse/config/raven/service-config.json.template index 7e586edf5..7325b63cf 100644 --- a/birdhouse/config/raven/service-config.json.template +++ b/birdhouse/config/raven/service-config.json.template @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", "name": "raven", "keywords": [ "service-wps" diff --git a/birdhouse/config/thredds/service-config.json.template b/birdhouse/config/thredds/service-config.json.template index 75cb55622..9cae617da 100644 --- a/birdhouse/config/thredds/service-config.json.template +++ b/birdhouse/config/thredds/service-config.json.template @@ -1,4 +1,5 @@ { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", "name": "thredds", "keywords": [ "data", From 1d75aa1090f9597dd934d78041b47af2de05eb51 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Thu, 15 Jun 2023 15:30:32 -0400 Subject: [PATCH 25/39] fix substitution error in tests --- tests/test_deployment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 1a034adb9..46a3d5379 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -43,7 +43,7 @@ def test_service_config_name_same_as_dirname(self, component_paths): service_config_file = os.path.join(path, "service-config.json.template") if os.path.isfile(service_config_file): with open(service_config_file) as f: - service_config = json.loads(Template(f.read()).substitute(TEMPLATE_SUBSTITUTIONS)) + service_config = json.loads(Template(f.read()).safe_substitute(TEMPLATE_SUBSTITUTIONS)) config_name = service_config.get("name") path_name = os.path.basename(path) if config_name != path_name: @@ -58,7 +58,7 @@ def test_service_config_valid(self, component_paths, services_config_schema): service_config_file = os.path.join(path, "service-config.json.template") if os.path.isfile(service_config_file): with open(service_config_file) as f: - service_config = json.loads(Template(f.read()).substitute(TEMPLATE_SUBSTITUTIONS)) + service_config = json.loads(Template(f.read()).safe_substitute(TEMPLATE_SUBSTITUTIONS)) try: jsonschema.validate(instance=service_config, schema=services_config_schema) except jsonschema.exceptions.ValidationError as e: From a5283539e645b97e275d33d31ed2641cdd46c263 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 15 Jun 2023 21:54:02 -0400 Subject: [PATCH 26/39] autodeploy testing doc: apply review feedback --- birdhouse/components/README.rst | 52 +++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index 73f3a4dce..6fa33838f 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -140,11 +140,11 @@ How to test platform autodeploy is not broken by a PR There are 2 tests that need to be performed: -* Can autodeploy deploy the PR from ``master`` branch, the stable reference point +* Can autodeploy deploy the PR from ``master`` branch, the stable reference point? - * This could fail if some changes in the PR, are incompatible with autodeploy. For example: ``./pavics-compose.sh`` calls some binaries that do not exist in the autodeploy docker image. + * This could fail if some changes in the PR are incompatible with autodeploy. For example: ``./pavics-compose.sh`` calls some binaries that do not exist in the autodeploy docker image. -* Can autodeploy be triggered again successfully, after the PR is live +* Can autodeploy be triggered again successfully, after the PR is live? * This could fail if the PR renamed some files and forgot to add the old file names to a ``.gitignore`` file. Then old file names will appear as new uncommitted files and autodeploy will halt because it expects a clean working directory. @@ -171,23 +171,29 @@ Here is a sample setup to test autodeploy: echo 'export EXTRA_CONF_DIRS="$EXTRA_CONF_DIRS ./components/scheduler" >> env.local # set AUTODEPLOY_PLATFORM_FREQUENCY + # can set to more frequent than 5 minutes if your machine is capable enough echo 'export AUTODEPLOY_PLATFORM_FREQUENCY="@every 5m"' >> env.local + # if scheduler container already running: # recreate scheduler container for new AUTODEPLOY_PLATFORM_FREQUENCY to be effective ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d -* Create a ``test`` branch so you can add bogus commits without affecting your real PR. Set up your main checkout (birdhouse-deploy) to track that test branch so it will detect new changes on the test branch and trigger the autodeploy. + # if scheduler container not running yet: start the newly added scheduler component + ./pavics-compose.sh up -d + +* Create a ``${USER}-test`` branch so you can add bogus commits without affecting your real PR. Set up your main checkout (birdhouse-deploy) to track that test branch so it will detect new changes on the test branch and trigger the autodeploy. .. code-block:: shell # go to the main checkout cd birdhouse-deploy/birdhouse - # initially create the test branch from master + # initially create the ${USER}-test branch from master + # the ${USER} prefix is to avoid name clash if another user is also testing autodeploy git checkout master git pull - git checkout -b test - git push -u test + git checkout -b ${USER}-test + git push -u ${USER}-test # ensure your runnings code is at "master" and is working correctly # if you do not have a working baseline, you will not know if the breakage is due to autodeploy or your code @@ -200,19 +206,27 @@ Here is a sample setup to test autodeploy: # go to the other checkout to trigger autodeploy cd birdhouse-deploy-trigger/birdhouse - # set branch test to the same commit as your PR, this will trigger autodeploy from master to your PR + # set branch ${USER}-test to the same commit as your PR, this will trigger autodeploy from master to your PR git pull - git checkout test + git checkout ${USER}-test git reset --hard YOUR_PR_BRANCH git push - # now that the remote "test" branch differs from the local "test" branch in the birdhouse-deploy repo, + # now that the remote "${USER}-test" branch differs from the local "${USER}-test" branch in the birdhouse-deploy repo, # the autodeploy mechanism will detect that the remote branch has changed and attempt to update the local branch # follow logs, check for errors tail -f /var/log/PAVICS/autodeploy.log - # do spot checks, run Jenkins on your deployment if needed + # each autodeploy trigger will start the log with + # ========== + # triggerdeploy START_TIME=2023-06-15T05:07:01+0000 + + # each autodeploy trigger will end the log with + # triggerdeploy finished START_TIME=2023-06-15T05:07:01+0000 + # triggerdeploy finished END_TIME=2023-06-15T05:07:06+0000 + + # do spot checks in the log, run Jenkins on your deployment if needed * Test scenario 2, from your PR to later changes @@ -227,30 +241,30 @@ Here is a sample setup to test autodeploy: git commit -m "trigger autodeploy" git push - # now that the remote "test" branch differs from the local "test" branch in the birdhouse-deploy repo, + # now that the remote "${USER}-test" branch differs from the local "${USER}-test" branch in the birdhouse-deploy repo, # the autodeploy mechanism will detect that the remote branch has changed and attempt to update the local branch # follow logs, check for errors tail -f /var/log/PAVICS/autodeploy.log -* Test done, clean up the bogus ``test`` branch and optionally relax AUTODEPLOY_PLATFORM_FREQUENCY +* Test done, clean up the bogus ``${USER}-test`` branch and optionally relax ``AUTODEPLOY_PLATFORM_FREQUENCY`` .. code-block:: shell # go to the other checkout to trigger autodeploy cd birdhouse-deploy-trigger/birdhouse - # go to master so we can delete the test branch + # go to master so we can delete the ${USER}-test branch git checkout master - git push origin --delete test - git branch -D test + git push origin --delete ${USER}-test + git branch -D ${USER}-test # go to the main checkout cd birdhouse-deploy/birdhouse - # go to YOUR_PR_BRANCH so we can delete the test branch + # go to YOUR_PR_BRANCH so we can delete the ${USER}-test branch git checkout YOUR_PR_BRANCH - git branch -D test + git branch -D ${USER}-test # edit env.local and change AUTODEPLOY_PLATFORM_FREQUENCY to something less frequent to save your cpu # do not remove the scheduler component from the stack yet or the next command will fail @@ -259,6 +273,8 @@ Here is a sample setup to test autodeploy: ./pavics-compose.sh stop scheduler && ./pavics-compose.sh rm -vf scheduler && ./pavics-compose.sh up -d # optionally edit env.local to remove the scheduler component from the stack + # then remove the running scheduler container + ./pavics-compose.sh up -d --remove-orphans Monitoring From d219046295006fdc299e32f64e765c2eb9827961 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 15 Jun 2023 22:02:37 -0400 Subject: [PATCH 27/39] environment-dev.yml: use the other tests/requirements.txt to avoid duplication Also to allow both Conda and Pip usage, depending on users's preference. --- environment-dev.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index aee5f7798..043adb5d3 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -4,7 +4,10 @@ channels: - conda-forge # for bump2version dependencies: - make + + # make bump-install will install bump2version using pip, for user prefering pip - bump2version - - pytest - - jsonschema - - requests + + - pip + - pip: + - -r tests/requirements.txt From 2cd43334c368b7d30f9db64e48166741aa1afa59 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 15 Jun 2023 23:53:33 -0400 Subject: [PATCH 28/39] tests: fix random error on some hosts only Fix the errors below: ============================= test session starts ============================== platform linux -- Python 3.11.4, pytest-7.2.2, pluggy-1.0.0 rootdir: /repos/birdhouse-deploy-WIP collected 13 items tests/test_read_configs_include.py .FF.F.FF.F.FF [100%] =================================== FAILURES =================================== ____________________ TestReadConfigs.test_all_conf_dirs_set ____________________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_set(self, read_config_include_file) -> None: """Test that the ALL_CONF_DIRS variable is set""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') print(proc.stdout) # useful when assert fail > assert get_command_stdout(proc).strip() E AssertionError: assert '' E + where '' = () E + where = '\n\n'.strip E + where '\n\n' = get_command_stdout(CompletedProcess(args='. /repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh && read_configs && echo ...store/repos/birdhouse-deploy-WIP\nUsing local environment file at: /tmp/tmpa6fffa6a\n#env for testing#\n\n', stderr='')) tests/test_read_configs_include.py:116: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmpa6fffa6a /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmpa6fffa6a _______________ TestReadConfigs.test_all_conf_dirs_default_order _______________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_default_order(self, read_config_include_file) -> None: """Test that the expected order that default.env files are loaded is correct""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc)) == self.default_all_conf_order E AssertionError: assert [] == ['./config/pr...leefowl', ...] E Right contains 23 more items, first extra item: './config/proxy' E Use -v to get more diff tests/test_read_configs_include.py:123: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmp6z8tjeb6 /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmp6z8tjeb6 ________________ TestReadConfigs.test_dependencies_loaded_first ________________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_dependencies_loaded_first(self, read_config_include_file) -> None: """Test that dependencies are loaded first""" extra = {"EXTRA_CONF_DIRS": '"./optional-components/test-weaver"'} proc = self.run_func(read_config_include_file, extra, 'echo "$ALL_CONF_DIRS"') print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc))[-2:] == [ "./components/weaver", "./optional-components/test-weaver", ] E AssertionError: assert ['./optional-.../test-weaver'] == ['./component.../test-weaver'] E At index 0 diff: './optional-components/test-weaver' != './components/weaver' E Right contains one more item: './optional-components/test-weaver' E Use -v to get more diff tests/test_read_configs_include.py:140: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmplo58uftq /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmplo58uftq ./optional-components/test-weaver _______________ TestReadConfigs.test_delayed_eval_default_value ________________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_default_value(self, read_config_include_file) -> None: """Test delayed eval when value not set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"'} proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') print(proc.stdout) # useful when assert fail # By default, PAVICS_FQDN_PUBLIC has same value as PAVICS_FQDN. > assert (split_and_strip(get_command_stdout(proc))[-1] == "fqdn.example.com - /data/jupyterhub_user_data - /data/geoserver") E AssertionError: assert '- /data/jupy...ata/geoserver' == 'fqdn.example...ata/geoserver' E - fqdn.example.com - /data/jupyterhub_user_data - /data/geoserver E ? ----------------- E + - /data/jupyterhub_user_data - /data/geoserver tests/test_read_configs_include.py:159: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmp1tjl5kuq /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmp1tjl5kuq - /data/jupyterhub_user_data - /data/geoserver ________________ TestReadConfigs.test_delayed_eval_custom_value ________________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_custom_value(self, read_config_include_file) -> None: """Test delayed eval when value is set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"', "PAVICS_FQDN_PUBLIC": '"public.example.com"', "DATA_PERSIST_ROOT": '"/my-data-root"', # indirectly change JUPYTERHUB_USER_DATA_DIR "GEOSERVER_DATA_DIR": '"/my-geoserver-data"', } proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') print(proc.stdout) # useful when assert fail # If PAVICS_FQDN_PUBLIC is set in env.local, that value should be effective. > assert (split_and_strip(get_command_stdout(proc))[-1] == "public.example.com - /my-data-root/jupyterhub_user_data - /my-geoserver-data") E AssertionError: assert 'public.examp...eoserver-data' == 'public.examp...eoserver-data' E - public.example.com - /my-data-root/jupyterhub_user_data - /my-geoserver-data E ? --- ----- E + public.example.com - /data/jupyterhub_user_data - /my-geoserver-data tests/test_read_configs_include.py:174: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmp001vbau5 /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmp001vbau5 public.example.com - /data/jupyterhub_user_data - /my-geoserver-data _____________ TestCreateComposeConfList.test_compose_no_overrides ______________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_no_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are no overrides""" proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/raven"}, 'echo "$COMPOSE_CONF_LIST"' ) print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", "./config/raven/docker-compose-extra.yml", ] E AssertionError: assert ['docker-compose.yml'] == ['docker-comp...se-extra.yml'] E Right contains 2 more items, first extra item: './config/finch/docker-compose-extra.yml' E Use -v to get more diff tests/test_read_configs_include.py:280: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse /repos/birdhouse-deploy-WIP -f docker-compose.yml _______________ TestCreateComposeConfList.test_compose_overrides _______________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are overrides""" proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/magpie"}, 'echo "$COMPOSE_CONF_LIST"' ) print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", "./config/magpie/docker-compose-extra.yml", "./config/finch/config/magpie/docker-compose-extra.yml", ] E AssertionError: assert ['docker-compose.yml'] == ['docker-comp...se-extra.yml'] E Right contains 3 more items, first extra item: './config/finch/docker-compose-extra.yml' E Use -v to get more diff tests/test_read_configs_include.py:305: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse /repos/birdhouse-deploy-WIP -f docker-compose.yml _____________ TestCreateComposeConfList.test_default_all_conf_dirs _____________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_default_all_conf_dirs(self, read_config_include_file): proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": " ".join(TestReadConfigs.default_all_conf_order)}, 'echo "$COMPOSE_CONF_LIST"', ) print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc), split_on="-f") == self.default_conf_list_order E AssertionError: assert ['docker-compose.yml'] == ['docker-comp...tra.yml', ...] E Right contains 66 more items, first extra item: './config/proxy/docker-compose-extra.yml' E Use -v to get more diff tests/test_read_configs_include.py:320: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse /repos/birdhouse-deploy-WIP -f docker-compose.yml =========================== short test summary info ============================ FAILED tests/test_read_configs_include.py::TestReadConfigs::test_all_conf_dirs_set FAILED tests/test_read_configs_include.py::TestReadConfigs::test_all_conf_dirs_default_order FAILED tests/test_read_configs_include.py::TestReadConfigs::test_dependencies_loaded_first FAILED tests/test_read_configs_include.py::TestReadConfigs::test_delayed_eval_default_value FAILED tests/test_read_configs_include.py::TestReadConfigs::test_delayed_eval_custom_value FAILED tests/test_read_configs_include.py::TestCreateComposeConfList::test_compose_no_overrides FAILED tests/test_read_configs_include.py::TestCreateComposeConfList::test_compose_overrides FAILED tests/test_read_configs_include.py::TestCreateComposeConfList::test_default_all_conf_dirs ========================= 8 failed, 5 passed in 0.40s ========================== --- CHANGES.md | 6 ++++++ tests/test_read_configs_include.py | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 34d1f1a81..72ce95038 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,12 @@ - Dev environment: add Conda `environment-dev.yml` to easily install all the dev tools required +- Test random error: fix test failure on some machine only + + The reason the tests fail is still unclear and illogical. Why only 8 of the + 13 fail but the same fix work for them all? Why the other 5 that did not + fail do not need that same fix? + [1.26.4](https://github.com/bird-house/birdhouse-deploy/tree/1.26.4) (2023-06-06) ------------------------------------------------------------------------------------------------------------------ diff --git a/tests/test_read_configs_include.py b/tests/test_read_configs_include.py index 8ce478875..71080244e 100644 --- a/tests/test_read_configs_include.py +++ b/tests/test_read_configs_include.py @@ -12,6 +12,15 @@ def root_dir(request): return os.path.dirname(os.path.dirname(request.fspath)) +@pytest.fixture(scope="function") +def run_in_compose_dir(root_dir): + compose_dir = os.path.join(root_dir, "birdhouse") + old_cwd = os.getcwd() + os.chdir(compose_dir) + yield + os.chdir(old_cwd) + + @pytest.fixture(scope="module") def read_config_include_file(root_dir) -> str: return os.path.join(root_dir, "birdhouse", "read-configs.include.sh") @@ -99,14 +108,18 @@ def test_return_code(self, read_config_include_file) -> None: proc = self.run_func(read_config_include_file, {}) assert proc.returncode == 0 + @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_set(self, read_config_include_file) -> None: """Test that the ALL_CONF_DIRS variable is set""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') + print(proc.stdout) # useful when assert fail assert get_command_stdout(proc).strip() + @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_default_order(self, read_config_include_file) -> None: """Test that the expected order that default.env files are loaded is correct""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc)) == self.default_all_conf_order def test_all_conf_dirs_extra_last(self, read_config_include_file) -> None: @@ -118,10 +131,12 @@ def test_all_conf_dirs_extra_last(self, read_config_include_file) -> None: "./components/weaver", ] + @pytest.mark.usefixtures("run_in_compose_dir") def test_dependencies_loaded_first(self, read_config_include_file) -> None: """Test that dependencies are loaded first""" extra = {"EXTRA_CONF_DIRS": '"./optional-components/test-weaver"'} proc = self.run_func(read_config_include_file, extra, 'echo "$ALL_CONF_DIRS"') + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc))[-2:] == [ "./components/weaver", "./optional-components/test-weaver", @@ -133,15 +148,18 @@ def test_non_project_components_included(self, read_config_include_file) -> None proc = self.run_func(read_config_include_file, extra, 'echo "$ALL_CONF_DIRS"') assert split_and_strip(get_command_stdout(proc))[-1] == "./blah/other-random-component" + @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_default_value(self, read_config_include_file) -> None: """Test delayed eval when value not set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"'} proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') + print(proc.stdout) # useful when assert fail # By default, PAVICS_FQDN_PUBLIC has same value as PAVICS_FQDN. assert (split_and_strip(get_command_stdout(proc))[-1] == "fqdn.example.com - /data/jupyterhub_user_data - /data/geoserver") + @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_custom_value(self, read_config_include_file) -> None: """Test delayed eval when value is set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"', @@ -151,6 +169,7 @@ def test_delayed_eval_custom_value(self, read_config_include_file) -> None: } proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') + print(proc.stdout) # useful when assert fail # If PAVICS_FQDN_PUBLIC is set in env.local, that value should be effective. assert (split_and_strip(get_command_stdout(proc))[-1] == "public.example.com - /my-data-root/jupyterhub_user_data - /my-geoserver-data") @@ -251,11 +270,13 @@ def test_all_conf_dirs_empty(self, read_config_include_file): proc = self.run_func(read_config_include_file, {}, 'echo "$COMPOSE_CONF_LIST"') assert split_and_strip(get_command_stdout(proc)) == ["-f docker-compose.yml"] + @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_no_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are no overrides""" proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/raven"}, 'echo "$COMPOSE_CONF_LIST"' ) + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", @@ -274,11 +295,13 @@ def test_compose_in_order(self, read_config_include_file): out2 = split_and_strip(get_command_stdout(proc2), split_on="-f") assert out1 == out2[:1] + out2[:0:-1] + @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are overrides""" proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/magpie"}, 'echo "$COMPOSE_CONF_LIST"' ) + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", @@ -286,10 +309,12 @@ def test_compose_overrides(self, read_config_include_file): "./config/finch/config/magpie/docker-compose-extra.yml", ] + @pytest.mark.usefixtures("run_in_compose_dir") def test_default_all_conf_dirs(self, read_config_include_file): proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": " ".join(TestReadConfigs.default_all_conf_order)}, 'echo "$COMPOSE_CONF_LIST"', ) + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == self.default_conf_list_order From 089bfed18f3f27a68dd6d5be59a432b6fff60302 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Fri, 16 Jun 2023 00:10:35 -0400 Subject: [PATCH 29/39] Revert "tests: fix random error on some hosts only" This reverts commit 2cd43334c368b7d30f9db64e48166741aa1afa59. --- CHANGES.md | 6 ------ tests/test_read_configs_include.py | 25 ------------------------- 2 files changed, 31 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 72ce95038..34d1f1a81 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,12 +20,6 @@ - Dev environment: add Conda `environment-dev.yml` to easily install all the dev tools required -- Test random error: fix test failure on some machine only - - The reason the tests fail is still unclear and illogical. Why only 8 of the - 13 fail but the same fix work for them all? Why the other 5 that did not - fail do not need that same fix? - [1.26.4](https://github.com/bird-house/birdhouse-deploy/tree/1.26.4) (2023-06-06) ------------------------------------------------------------------------------------------------------------------ diff --git a/tests/test_read_configs_include.py b/tests/test_read_configs_include.py index 71080244e..8ce478875 100644 --- a/tests/test_read_configs_include.py +++ b/tests/test_read_configs_include.py @@ -12,15 +12,6 @@ def root_dir(request): return os.path.dirname(os.path.dirname(request.fspath)) -@pytest.fixture(scope="function") -def run_in_compose_dir(root_dir): - compose_dir = os.path.join(root_dir, "birdhouse") - old_cwd = os.getcwd() - os.chdir(compose_dir) - yield - os.chdir(old_cwd) - - @pytest.fixture(scope="module") def read_config_include_file(root_dir) -> str: return os.path.join(root_dir, "birdhouse", "read-configs.include.sh") @@ -108,18 +99,14 @@ def test_return_code(self, read_config_include_file) -> None: proc = self.run_func(read_config_include_file, {}) assert proc.returncode == 0 - @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_set(self, read_config_include_file) -> None: """Test that the ALL_CONF_DIRS variable is set""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') - print(proc.stdout) # useful when assert fail assert get_command_stdout(proc).strip() - @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_default_order(self, read_config_include_file) -> None: """Test that the expected order that default.env files are loaded is correct""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') - print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc)) == self.default_all_conf_order def test_all_conf_dirs_extra_last(self, read_config_include_file) -> None: @@ -131,12 +118,10 @@ def test_all_conf_dirs_extra_last(self, read_config_include_file) -> None: "./components/weaver", ] - @pytest.mark.usefixtures("run_in_compose_dir") def test_dependencies_loaded_first(self, read_config_include_file) -> None: """Test that dependencies are loaded first""" extra = {"EXTRA_CONF_DIRS": '"./optional-components/test-weaver"'} proc = self.run_func(read_config_include_file, extra, 'echo "$ALL_CONF_DIRS"') - print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc))[-2:] == [ "./components/weaver", "./optional-components/test-weaver", @@ -148,18 +133,15 @@ def test_non_project_components_included(self, read_config_include_file) -> None proc = self.run_func(read_config_include_file, extra, 'echo "$ALL_CONF_DIRS"') assert split_and_strip(get_command_stdout(proc))[-1] == "./blah/other-random-component" - @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_default_value(self, read_config_include_file) -> None: """Test delayed eval when value not set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"'} proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') - print(proc.stdout) # useful when assert fail # By default, PAVICS_FQDN_PUBLIC has same value as PAVICS_FQDN. assert (split_and_strip(get_command_stdout(proc))[-1] == "fqdn.example.com - /data/jupyterhub_user_data - /data/geoserver") - @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_custom_value(self, read_config_include_file) -> None: """Test delayed eval when value is set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"', @@ -169,7 +151,6 @@ def test_delayed_eval_custom_value(self, read_config_include_file) -> None: } proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') - print(proc.stdout) # useful when assert fail # If PAVICS_FQDN_PUBLIC is set in env.local, that value should be effective. assert (split_and_strip(get_command_stdout(proc))[-1] == "public.example.com - /my-data-root/jupyterhub_user_data - /my-geoserver-data") @@ -270,13 +251,11 @@ def test_all_conf_dirs_empty(self, read_config_include_file): proc = self.run_func(read_config_include_file, {}, 'echo "$COMPOSE_CONF_LIST"') assert split_and_strip(get_command_stdout(proc)) == ["-f docker-compose.yml"] - @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_no_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are no overrides""" proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/raven"}, 'echo "$COMPOSE_CONF_LIST"' ) - print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", @@ -295,13 +274,11 @@ def test_compose_in_order(self, read_config_include_file): out2 = split_and_strip(get_command_stdout(proc2), split_on="-f") assert out1 == out2[:1] + out2[:0:-1] - @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are overrides""" proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/magpie"}, 'echo "$COMPOSE_CONF_LIST"' ) - print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", @@ -309,12 +286,10 @@ def test_compose_overrides(self, read_config_include_file): "./config/finch/config/magpie/docker-compose-extra.yml", ] - @pytest.mark.usefixtures("run_in_compose_dir") def test_default_all_conf_dirs(self, read_config_include_file): proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": " ".join(TestReadConfigs.default_all_conf_order)}, 'echo "$COMPOSE_CONF_LIST"', ) - print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == self.default_conf_list_order From 449c36a78ba20f8fa7bb420f0886ea74d00853b3 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Fri, 16 Jun 2023 00:33:30 -0400 Subject: [PATCH 30/39] read-configs: fix bugs in COMPOSE_DIR detection To fix the following failure due to checkout dir is not named "birdhouse-deploy". ``` ============================= test session starts ============================== platform linux -- Python 3.11.4, pytest-7.2.2, pluggy-1.0.0 rootdir: /repos/birdhouse-deploy-WIP collected 13 items tests/test_read_configs_include.py .FF.F.FF.F.FF [100%] =================================== FAILURES =================================== ____________________ TestReadConfigs.test_all_conf_dirs_set ____________________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_set(self, read_config_include_file) -> None: """Test that the ALL_CONF_DIRS variable is set""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') print(proc.stdout) # useful when assert fail > assert get_command_stdout(proc).strip() E AssertionError: assert '' E + where '' = () E + where = '\n\n'.strip E + where '\n\n' = get_command_stdout(CompletedProcess(args='. /repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh && read_configs && echo ...store/repos/birdhouse-deploy-WIP\nUsing local environment file at: /tmp/tmpa6fffa6a\n#env for testing#\n\n', stderr='')) tests/test_read_configs_include.py:116: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmpa6fffa6a /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmpa6fffa6a #env for testing# _______________ TestReadConfigs.test_all_conf_dirs_default_order _______________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_default_order(self, read_config_include_file) -> None: """Test that the expected order that default.env files are loaded is correct""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc)) == self.default_all_conf_order E AssertionError: assert [] == ['./config/pr...leefowl', ...] E Right contains 23 more items, first extra item: './config/proxy' E Use -v to get more diff tests/test_read_configs_include.py:123: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmp6z8tjeb6 /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmp6z8tjeb6 #env for testing# ________________ TestReadConfigs.test_dependencies_loaded_first ________________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_dependencies_loaded_first(self, read_config_include_file) -> None: """Test that dependencies are loaded first""" extra = {"EXTRA_CONF_DIRS": '"./optional-components/test-weaver"'} proc = self.run_func(read_config_include_file, extra, 'echo "$ALL_CONF_DIRS"') print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc))[-2:] == [ "./components/weaver", "./optional-components/test-weaver", ] E AssertionError: assert ['./optional-.../test-weaver'] == ['./component.../test-weaver'] E At index 0 diff: './optional-components/test-weaver' != './components/weaver' E Right contains one more item: './optional-components/test-weaver' E Use -v to get more diff tests/test_read_configs_include.py:140: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmplo58uftq /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmplo58uftq #env for testing# ./optional-components/test-weaver _______________ TestReadConfigs.test_delayed_eval_default_value ________________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_default_value(self, read_config_include_file) -> None: """Test delayed eval when value not set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"'} proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') print(proc.stdout) # useful when assert fail # By default, PAVICS_FQDN_PUBLIC has same value as PAVICS_FQDN. > assert (split_and_strip(get_command_stdout(proc))[-1] == "fqdn.example.com - /data/jupyterhub_user_data - /data/geoserver") E AssertionError: assert '- /data/jupy...ata/geoserver' == 'fqdn.example...ata/geoserver' E - fqdn.example.com - /data/jupyterhub_user_data - /data/geoserver E ? ----------------- E + - /data/jupyterhub_user_data - /data/geoserver tests/test_read_configs_include.py:159: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmp1tjl5kuq /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmp1tjl5kuq #env for testing# - /data/jupyterhub_user_data - /data/geoserver ________________ TestReadConfigs.test_delayed_eval_custom_value ________________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_custom_value(self, read_config_include_file) -> None: """Test delayed eval when value is set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"', "PAVICS_FQDN_PUBLIC": '"public.example.com"', "DATA_PERSIST_ROOT": '"/my-data-root"', # indirectly change JUPYTERHUB_USER_DATA_DIR "GEOSERVER_DATA_DIR": '"/my-geoserver-data"', } proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') print(proc.stdout) # useful when assert fail # If PAVICS_FQDN_PUBLIC is set in env.local, that value should be effective. > assert (split_and_strip(get_command_stdout(proc))[-1] == "public.example.com - /my-data-root/jupyterhub_user_data - /my-geoserver-data") E AssertionError: assert 'public.examp...eoserver-data' == 'public.examp...eoserver-data' E - public.example.com - /my-data-root/jupyterhub_user_data - /my-geoserver-data E ? --- ----- E + public.example.com - /data/jupyterhub_user_data - /my-geoserver-data tests/test_read_configs_include.py:174: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse Using local environment file at: /tmp/tmp001vbau5 /repos/birdhouse-deploy-WIP Using local environment file at: /tmp/tmp001vbau5 #env for testing# public.example.com - /data/jupyterhub_user_data - /my-geoserver-data _____________ TestCreateComposeConfList.test_compose_no_overrides ______________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_no_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are no overrides""" proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/raven"}, 'echo "$COMPOSE_CONF_LIST"' ) print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", "./config/raven/docker-compose-extra.yml", ] E AssertionError: assert ['docker-compose.yml'] == ['docker-comp...se-extra.yml'] E Right contains 2 more items, first extra item: './config/finch/docker-compose-extra.yml' E Use -v to get more diff tests/test_read_configs_include.py:280: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse /repos/birdhouse-deploy-WIP #env for testing# -f docker-compose.yml _______________ TestCreateComposeConfList.test_compose_overrides _______________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are overrides""" proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/magpie"}, 'echo "$COMPOSE_CONF_LIST"' ) print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", "./config/magpie/docker-compose-extra.yml", "./config/finch/config/magpie/docker-compose-extra.yml", ] E AssertionError: assert ['docker-compose.yml'] == ['docker-comp...se-extra.yml'] E Right contains 3 more items, first extra item: './config/finch/docker-compose-extra.yml' E Use -v to get more diff tests/test_read_configs_include.py:305: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse /repos/birdhouse-deploy-WIP #env for testing# -f docker-compose.yml _____________ TestCreateComposeConfList.test_default_all_conf_dirs _____________ self = read_config_include_file = '/repos/birdhouse-deploy-WIP/birdhouse/read-configs.include.sh' @pytest.mark.usefixtures("run_in_compose_dir") def test_default_all_conf_dirs(self, read_config_include_file): proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": " ".join(TestReadConfigs.default_all_conf_order)}, 'echo "$COMPOSE_CONF_LIST"', ) print(proc.stdout) # useful when assert fail > assert split_and_strip(get_command_stdout(proc), split_on="-f") == self.default_conf_list_order E AssertionError: assert ['docker-compose.yml'] == ['docker-comp...tra.yml', ...] E Right contains 66 more items, first extra item: './config/proxy/docker-compose-extra.yml' E Use -v to get more diff tests/test_read_configs_include.py:320: AssertionError ----------------------------- Captured stdout call ----------------------------- /repos/birdhouse-deploy/birdhouse /repos/birdhouse-deploy-WIP #env for testing# -f docker-compose.yml =========================== short test summary info ============================ FAILED tests/test_read_configs_include.py::TestReadConfigs::test_all_conf_dirs_set FAILED tests/test_read_configs_include.py::TestReadConfigs::test_all_conf_dirs_default_order FAILED tests/test_read_configs_include.py::TestReadConfigs::test_dependencies_loaded_first FAILED tests/test_read_configs_include.py::TestReadConfigs::test_delayed_eval_default_value FAILED tests/test_read_configs_include.py::TestReadConfigs::test_delayed_eval_custom_value FAILED tests/test_read_configs_include.py::TestCreateComposeConfList::test_compose_no_overrides FAILED tests/test_read_configs_include.py::TestCreateComposeConfList::test_compose_overrides FAILED tests/test_read_configs_include.py::TestCreateComposeConfList::test_default_all_conf_dirs ========================= 8 failed, 5 passed in 0.40s ========================== ``` --- CHANGES.md | 7 +++++++ birdhouse/read-configs.include.sh | 13 ++++++++++--- tests/test_read_configs_include.py | 8 ++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 34d1f1a81..18f2f9067 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,13 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ +## Fixes +- Tests: some tests fail to run when `CWD` is not `COMPOSE_DIR` + + The root cause is the automatic `COMPOSE_DIR` detection in + `read-configs.include.sh` missed one case and the detection ordering was wrong + for one other case as well. + ## Changes - Autodeploy: document test procedure diff --git a/birdhouse/read-configs.include.sh b/birdhouse/read-configs.include.sh index bdc214581..4c5578bea 100644 --- a/birdhouse/read-configs.include.sh +++ b/birdhouse/read-configs.include.sh @@ -25,6 +25,7 @@ # Derive COMPOSE_DIR from the most probable locations. # This is NOT meant to be exhautive. +# Assume the checkout is named "birdhouse-deploy", which might NOT be true. # Caller of this file can simply set COMPOSE_DIR itself. discover_compose_dir() { if [ -z "$COMPOSE_DIR" ] || [ ! -e "$COMPOSE_DIR" ]; then @@ -35,6 +36,15 @@ discover_compose_dir() { # Parent dir is COMPOSE_DIR # Case of all the scripts under deployment/ or scripts/ COMPOSE_DIR="$(realpath ..)" + elif [ -f "../birdhouse/pavics-compose.sh" ]; then + # Sibling dir is COMPOSE_DIR + # Case of all the tests under tests/ + COMPOSE_DIR="$(realpath ../birdhouse)" + elif [ -f "./birdhouse/pavics-compose.sh" ]; then + # Child dir is COMPOSE_DIR + COMPOSE_DIR="$(realpath birdhouse)" + # Below assume checkout is named birdhouse-deploy, which might not + # always be true. elif [ -f "../birdhouse-deploy/birdhouse/pavics-compose.sh" ]; then # Case of sibling checkout at same level as birdhouse-deploy. COMPOSE_DIR="$(realpath "../birdhouse-deploy/birdhouse")" @@ -44,9 +54,6 @@ discover_compose_dir() { elif [ -f "../../../birdhouse-deploy/birdhouse/pavics-compose.sh" ]; then # Case of sub-subdir of sibling checkout at same level as birdhouse-deploy. COMPOSE_DIR="$(realpath "../../../birdhouse-deploy/birdhouse")" - elif [ -f "./birdhouse/pavics-compose.sh" ]; then - # Child dir is COMPOSE_DIR - COMPOSE_DIR="$(realpath birdhouse)" fi echo "$COMPOSE_DIR" export COMPOSE_DIR diff --git a/tests/test_read_configs_include.py b/tests/test_read_configs_include.py index 8ce478875..06dbedf80 100644 --- a/tests/test_read_configs_include.py +++ b/tests/test_read_configs_include.py @@ -102,11 +102,13 @@ def test_return_code(self, read_config_include_file) -> None: def test_all_conf_dirs_set(self, read_config_include_file) -> None: """Test that the ALL_CONF_DIRS variable is set""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') + print(proc.stdout) # useful when assert fail assert get_command_stdout(proc).strip() def test_all_conf_dirs_default_order(self, read_config_include_file) -> None: """Test that the expected order that default.env files are loaded is correct""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc)) == self.default_all_conf_order def test_all_conf_dirs_extra_last(self, read_config_include_file) -> None: @@ -122,6 +124,7 @@ def test_dependencies_loaded_first(self, read_config_include_file) -> None: """Test that dependencies are loaded first""" extra = {"EXTRA_CONF_DIRS": '"./optional-components/test-weaver"'} proc = self.run_func(read_config_include_file, extra, 'echo "$ALL_CONF_DIRS"') + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc))[-2:] == [ "./components/weaver", "./optional-components/test-weaver", @@ -138,6 +141,7 @@ def test_delayed_eval_default_value(self, read_config_include_file) -> None: extra = {"PAVICS_FQDN": '"fqdn.example.com"'} proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') + print(proc.stdout) # useful when assert fail # By default, PAVICS_FQDN_PUBLIC has same value as PAVICS_FQDN. assert (split_and_strip(get_command_stdout(proc))[-1] == "fqdn.example.com - /data/jupyterhub_user_data - /data/geoserver") @@ -151,6 +155,7 @@ def test_delayed_eval_custom_value(self, read_config_include_file) -> None: } proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') + print(proc.stdout) # useful when assert fail # If PAVICS_FQDN_PUBLIC is set in env.local, that value should be effective. assert (split_and_strip(get_command_stdout(proc))[-1] == "public.example.com - /my-data-root/jupyterhub_user_data - /my-geoserver-data") @@ -256,6 +261,7 @@ def test_compose_no_overrides(self, read_config_include_file): proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/raven"}, 'echo "$COMPOSE_CONF_LIST"' ) + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", @@ -279,6 +285,7 @@ def test_compose_overrides(self, read_config_include_file): proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/magpie"}, 'echo "$COMPOSE_CONF_LIST"' ) + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", @@ -292,4 +299,5 @@ def test_default_all_conf_dirs(self, read_config_include_file): {"ALL_CONF_DIRS": " ".join(TestReadConfigs.default_all_conf_order)}, 'echo "$COMPOSE_CONF_LIST"', ) + print(proc.stdout) # useful when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == self.default_conf_list_order From 952be7e591d7d98fbcf60645ac59e8ec11eaf50e Mon Sep 17 00:00:00 2001 From: Long Vu Date: Fri, 16 Jun 2023 00:57:34 -0400 Subject: [PATCH 31/39] tests: ensure can be run from any CWD, even outside of checkout --- CHANGES.md | 6 ++++++ birdhouse/read-configs.include.sh | 2 +- tests/test_read_configs_include.py | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 18f2f9067..73cd2d899 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,6 +27,12 @@ - Dev environment: add Conda `environment-dev.yml` to easily install all the dev tools required +- Tests: make test runs more robust, able to run from any `CWD` + + Before, test runs can only be started from inside the checkout, at some + "popular" locations inside the checkout. Now it can be started from + litterally anywhere. + [1.26.4](https://github.com/bird-house/birdhouse-deploy/tree/1.26.4) (2023-06-06) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/read-configs.include.sh b/birdhouse/read-configs.include.sh index 4c5578bea..85f4950dc 100644 --- a/birdhouse/read-configs.include.sh +++ b/birdhouse/read-configs.include.sh @@ -26,7 +26,7 @@ # Derive COMPOSE_DIR from the most probable locations. # This is NOT meant to be exhautive. # Assume the checkout is named "birdhouse-deploy", which might NOT be true. -# Caller of this file can simply set COMPOSE_DIR itself. +# Caller of this file can simply set COMPOSE_DIR itself, this is the safest way. discover_compose_dir() { if [ -z "$COMPOSE_DIR" ] || [ ! -e "$COMPOSE_DIR" ]; then if [ -f "./pavics-compose.sh" ]; then diff --git a/tests/test_read_configs_include.py b/tests/test_read_configs_include.py index 06dbedf80..71080244e 100644 --- a/tests/test_read_configs_include.py +++ b/tests/test_read_configs_include.py @@ -12,6 +12,15 @@ def root_dir(request): return os.path.dirname(os.path.dirname(request.fspath)) +@pytest.fixture(scope="function") +def run_in_compose_dir(root_dir): + compose_dir = os.path.join(root_dir, "birdhouse") + old_cwd = os.getcwd() + os.chdir(compose_dir) + yield + os.chdir(old_cwd) + + @pytest.fixture(scope="module") def read_config_include_file(root_dir) -> str: return os.path.join(root_dir, "birdhouse", "read-configs.include.sh") @@ -99,12 +108,14 @@ def test_return_code(self, read_config_include_file) -> None: proc = self.run_func(read_config_include_file, {}) assert proc.returncode == 0 + @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_set(self, read_config_include_file) -> None: """Test that the ALL_CONF_DIRS variable is set""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') print(proc.stdout) # useful when assert fail assert get_command_stdout(proc).strip() + @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_default_order(self, read_config_include_file) -> None: """Test that the expected order that default.env files are loaded is correct""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') @@ -120,6 +131,7 @@ def test_all_conf_dirs_extra_last(self, read_config_include_file) -> None: "./components/weaver", ] + @pytest.mark.usefixtures("run_in_compose_dir") def test_dependencies_loaded_first(self, read_config_include_file) -> None: """Test that dependencies are loaded first""" extra = {"EXTRA_CONF_DIRS": '"./optional-components/test-weaver"'} @@ -136,6 +148,7 @@ def test_non_project_components_included(self, read_config_include_file) -> None proc = self.run_func(read_config_include_file, extra, 'echo "$ALL_CONF_DIRS"') assert split_and_strip(get_command_stdout(proc))[-1] == "./blah/other-random-component" + @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_default_value(self, read_config_include_file) -> None: """Test delayed eval when value not set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"'} @@ -146,6 +159,7 @@ def test_delayed_eval_default_value(self, read_config_include_file) -> None: assert (split_and_strip(get_command_stdout(proc))[-1] == "fqdn.example.com - /data/jupyterhub_user_data - /data/geoserver") + @pytest.mark.usefixtures("run_in_compose_dir") def test_delayed_eval_custom_value(self, read_config_include_file) -> None: """Test delayed eval when value is set in env.local""" extra = {"PAVICS_FQDN": '"fqdn.example.com"', @@ -256,6 +270,7 @@ def test_all_conf_dirs_empty(self, read_config_include_file): proc = self.run_func(read_config_include_file, {}, 'echo "$COMPOSE_CONF_LIST"') assert split_and_strip(get_command_stdout(proc)) == ["-f docker-compose.yml"] + @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_no_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are no overrides""" proc = self.run_func( @@ -280,6 +295,7 @@ def test_compose_in_order(self, read_config_include_file): out2 = split_and_strip(get_command_stdout(proc2), split_on="-f") assert out1 == out2[:1] + out2[:0:-1] + @pytest.mark.usefixtures("run_in_compose_dir") def test_compose_overrides(self, read_config_include_file): """Test that COMPOSE_CONF_LIST is set correctly when there are overrides""" proc = self.run_func( @@ -293,6 +309,7 @@ def test_compose_overrides(self, read_config_include_file): "./config/finch/config/magpie/docker-compose-extra.yml", ] + @pytest.mark.usefixtures("run_in_compose_dir") def test_default_all_conf_dirs(self, read_config_include_file): proc = self.run_func( read_config_include_file, From a90a1ab9e0f94fd7b17137898fafd14bd5d5e8fa Mon Sep 17 00:00:00 2001 From: Long Vu Date: Fri, 16 Jun 2023 09:49:25 -0400 Subject: [PATCH 32/39] test robustness: apply review feedback from @mishaschwartz --- tests/test_read_configs_include.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_read_configs_include.py b/tests/test_read_configs_include.py index 71080244e..e849d1ead 100644 --- a/tests/test_read_configs_include.py +++ b/tests/test_read_configs_include.py @@ -17,8 +17,10 @@ def run_in_compose_dir(root_dir): compose_dir = os.path.join(root_dir, "birdhouse") old_cwd = os.getcwd() os.chdir(compose_dir) - yield - os.chdir(old_cwd) + try: + yield + finally: + os.chdir(old_cwd) @pytest.fixture(scope="module") @@ -112,14 +114,14 @@ def test_return_code(self, read_config_include_file) -> None: def test_all_conf_dirs_set(self, read_config_include_file) -> None: """Test that the ALL_CONF_DIRS variable is set""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') - print(proc.stdout) # useful when assert fail + print(proc.stdout) # useful for debugging when assert fail assert get_command_stdout(proc).strip() @pytest.mark.usefixtures("run_in_compose_dir") def test_all_conf_dirs_default_order(self, read_config_include_file) -> None: """Test that the expected order that default.env files are loaded is correct""" proc = self.run_func(read_config_include_file, {}, 'echo "$ALL_CONF_DIRS"') - print(proc.stdout) # useful when assert fail + print(proc.stdout) # useful for debugging when assert fail assert split_and_strip(get_command_stdout(proc)) == self.default_all_conf_order def test_all_conf_dirs_extra_last(self, read_config_include_file) -> None: @@ -136,7 +138,7 @@ def test_dependencies_loaded_first(self, read_config_include_file) -> None: """Test that dependencies are loaded first""" extra = {"EXTRA_CONF_DIRS": '"./optional-components/test-weaver"'} proc = self.run_func(read_config_include_file, extra, 'echo "$ALL_CONF_DIRS"') - print(proc.stdout) # useful when assert fail + print(proc.stdout) # useful for debugging when assert fail assert split_and_strip(get_command_stdout(proc))[-2:] == [ "./components/weaver", "./optional-components/test-weaver", @@ -154,7 +156,7 @@ def test_delayed_eval_default_value(self, read_config_include_file) -> None: extra = {"PAVICS_FQDN": '"fqdn.example.com"'} proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') - print(proc.stdout) # useful when assert fail + print(proc.stdout) # useful for debugging when assert fail # By default, PAVICS_FQDN_PUBLIC has same value as PAVICS_FQDN. assert (split_and_strip(get_command_stdout(proc))[-1] == "fqdn.example.com - /data/jupyterhub_user_data - /data/geoserver") @@ -169,7 +171,7 @@ def test_delayed_eval_custom_value(self, read_config_include_file) -> None: } proc = self.run_func(read_config_include_file, extra, 'echo "$PAVICS_FQDN_PUBLIC - $JUPYTERHUB_USER_DATA_DIR - $GEOSERVER_DATA_DIR"') - print(proc.stdout) # useful when assert fail + print(proc.stdout) # useful for debugging when assert fail # If PAVICS_FQDN_PUBLIC is set in env.local, that value should be effective. assert (split_and_strip(get_command_stdout(proc))[-1] == "public.example.com - /my-data-root/jupyterhub_user_data - /my-geoserver-data") @@ -276,7 +278,7 @@ def test_compose_no_overrides(self, read_config_include_file): proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/raven"}, 'echo "$COMPOSE_CONF_LIST"' ) - print(proc.stdout) # useful when assert fail + print(proc.stdout) # useful for debugging when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", @@ -301,7 +303,7 @@ def test_compose_overrides(self, read_config_include_file): proc = self.run_func( read_config_include_file, {"ALL_CONF_DIRS": "./config/finch ./config/magpie"}, 'echo "$COMPOSE_CONF_LIST"' ) - print(proc.stdout) # useful when assert fail + print(proc.stdout) # useful for debugging when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == [ "docker-compose.yml", "./config/finch/docker-compose-extra.yml", @@ -316,5 +318,5 @@ def test_default_all_conf_dirs(self, read_config_include_file): {"ALL_CONF_DIRS": " ".join(TestReadConfigs.default_all_conf_order)}, 'echo "$COMPOSE_CONF_LIST"', ) - print(proc.stdout) # useful when assert fail + print(proc.stdout) # useful for debugging when assert fail assert split_and_strip(get_command_stdout(proc), split_on="-f") == self.default_conf_list_order From e9dc0efc651a7c7b6cd5d5161b626b03cce73dea Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Fri, 16 Jun 2023 13:01:20 -0400 Subject: [PATCH 33/39] =?UTF-8?q?Bump=20version:=201.26.4=20=E2=86=92=201.?= =?UTF-8?q?26.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 6 +++--- CHANGES.md | 5 +++++ Makefile | 2 +- README.rst | 8 ++++---- RELEASE.txt | 2 +- .../config/canarie-api/docker_configuration.py.template | 8 ++++---- docs/source/conf.py | 4 ++-- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 465e27745..ba72d59b1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.26.4 +current_version = 1.26.5 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-06-06T18:08:27Z +search = {current_version} 2023-06-16T17:01:20Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-06-06T18:08:27Z +values = 2023-06-16T17:01:20Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index e472d51fb..80248d276 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,11 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + +[1.26.5](https://github.com/bird-house/birdhouse-deploy/tree/1.26.5) (2023-06-16) +------------------------------------------------------------------------------------------------------------------ + ## Fixes - Autodeploy: optionally fix file permissions diff --git a/Makefile b/Makefile index 421dffcc2..f51a17aca 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.26.4 +override APP_VERSION := 1.26.5 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index 672d2fb9d..b5694125d 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.4.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.5.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.4...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.5...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.4-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.5-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.4 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.5 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index 1968b5629..ff9dbe4ba 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.26.4 2023-06-06T18:08:27Z +1.26.5 2023-06-16T17:01:20Z diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index d6d3e360d..e79785076 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.4', - 'releaseTime': '2023-06-06T18:08:27Z', + 'version': '1.26.5', + 'releaseTime': '2023-06-16T17:01:20Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.4', - 'releaseTime': '2023-06-06T18:08:27Z', + 'version': '1.26.5', + 'releaseTime': '2023-06-16T17:01:20Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/docs/source/conf.py b/docs/source/conf.py index 1e1752b72..e50d332f5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.26.4' +version = '1.26.5' # The full version, including alpha/beta/rc tags. -release = '1.26.4' +release = '1.26.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 0cb9d0fbc2d701cb5f57e7c0c25221bd67e69555 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Fri, 16 Jun 2023 13:06:27 -0400 Subject: [PATCH 34/39] =?UTF-8?q?Bump=20version:=201.26.5=20=E2=86=92=201.?= =?UTF-8?q?26.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 6 +++--- CHANGES.md | 5 +++++ Makefile | 2 +- README.rst | 8 ++++---- RELEASE.txt | 2 +- .../config/canarie-api/docker_configuration.py.template | 8 ++++---- docs/source/conf.py | 4 ++-- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ba72d59b1..a660c61e1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.26.5 +current_version = 1.26.6 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-06-16T17:01:20Z +search = {current_version} 2023-06-16T17:06:27Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-06-16T17:01:20Z +values = 2023-06-16T17:06:27Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index c7ccb81ba..882612ab3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,11 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + +[1.26.6](https://github.com/bird-house/birdhouse-deploy/tree/1.26.6) (2023-06-16) +------------------------------------------------------------------------------------------------------------------ + ## Fixes - `components/` endpoint displays intended information after auto-deploy diff --git a/Makefile b/Makefile index f51a17aca..b2c4ba034 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.26.5 +override APP_VERSION := 1.26.6 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index b5694125d..aaeb469c5 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.5.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.6.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.5...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.6...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.5-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.6-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.5 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.6 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index ff9dbe4ba..a4d2f1b88 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.26.5 2023-06-16T17:01:20Z +1.26.6 2023-06-16T17:06:27Z diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index e79785076..6697066b1 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.5', - 'releaseTime': '2023-06-16T17:01:20Z', + 'version': '1.26.6', + 'releaseTime': '2023-06-16T17:06:27Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.5', - 'releaseTime': '2023-06-16T17:01:20Z', + 'version': '1.26.6', + 'releaseTime': '2023-06-16T17:06:27Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/docs/source/conf.py b/docs/source/conf.py index e50d332f5..e0204e9ce 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.26.5' +version = '1.26.6' # The full version, including alpha/beta/rc tags. -release = '1.26.5' +release = '1.26.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From eca99b3b119e879bb40656591168d8c11f5116c2 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Fri, 16 Jun 2023 13:09:16 -0400 Subject: [PATCH 35/39] CHANGES: clarify why test failure was not caught before --- CHANGES.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 73cd2d899..391bf8744 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,9 +18,16 @@ ## Fixes - Tests: some tests fail to run when `CWD` is not `COMPOSE_DIR` - The root cause is the automatic `COMPOSE_DIR` detection in - `read-configs.include.sh` missed one case and the detection ordering was wrong - for one other case as well. + The root cause is the automatic `COMPOSE_DIR` detection in + `read-configs.include.sh` missed one case and the detection ordering was wrong + for one other case as well. + + This was not found before because the checkout was properly named + "birdhouse-deploy". When the checkout is named something else, then we hit + this error. + + Fixes the error found here + https://github.com/bird-house/birdhouse-deploy/pull/329#pullrequestreview-1480211502 ## Changes - Autodeploy: document test procedure From faf0f5cf2c2f2832627175330d026ae9d3eb2ea1 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Mon, 19 Jun 2023 16:23:19 -0400 Subject: [PATCH 36/39] =?UTF-8?q?Bump=20version:=201.26.6=20=E2=86=92=201.?= =?UTF-8?q?26.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 6 +++--- CHANGES.md | 5 +++++ Makefile | 2 +- README.rst | 8 ++++---- RELEASE.txt | 2 +- .../config/canarie-api/docker_configuration.py.template | 8 ++++---- docs/source/conf.py | 4 ++-- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a660c61e1..45e0c4cc5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.26.6 +current_version = 1.26.7 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-06-16T17:06:27Z +search = {current_version} 2023-06-19T20:23:19Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-06-16T17:06:27Z +values = 2023-06-19T20:23:19Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index 4f59d3cc0..97c446ad1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,11 @@ [//]: # (list changes here, using '-' for each new entry, remove this when items are added) +[1.26.7](https://github.com/bird-house/birdhouse-deploy/tree/1.26.7) (2023-06-19) +------------------------------------------------------------------------------------------------------------------ + +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + [1.26.6](https://github.com/bird-house/birdhouse-deploy/tree/1.26.6) (2023-06-16) ------------------------------------------------------------------------------------------------------------------ diff --git a/Makefile b/Makefile index b2c4ba034..e50f20190 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.26.6 +override APP_VERSION := 1.26.7 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index aaeb469c5..175458a38 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.6.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.7.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.6...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.7...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.6-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.7-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.6 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.7 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index a4d2f1b88..f96dae4e8 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.26.6 2023-06-16T17:06:27Z +1.26.7 2023-06-19T20:23:19Z diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index 6697066b1..ba2bcb583 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.6', - 'releaseTime': '2023-06-16T17:06:27Z', + 'version': '1.26.7', + 'releaseTime': '2023-06-19T20:23:19Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.6', - 'releaseTime': '2023-06-16T17:06:27Z', + 'version': '1.26.7', + 'releaseTime': '2023-06-19T20:23:19Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/docs/source/conf.py b/docs/source/conf.py index e0204e9ce..8e861e52e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.26.6' +version = '1.26.7' # The full version, including alpha/beta/rc tags. -release = '1.26.6' +release = '1.26.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 9f7934c2817112d7622df7e07d374c4a9915a011 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Mon, 19 Jun 2023 16:29:00 -0400 Subject: [PATCH 37/39] fix CHANGES.md update --- CHANGES.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 97c446ad1..a1e66d1f9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,7 +20,18 @@ [1.26.7](https://github.com/bird-house/birdhouse-deploy/tree/1.26.7) (2023-06-19) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Changes + +- A new endpoint `/services` is added that provides a JSON string describing each of the user facing services currently + enabled on the stack. This is a static string and serves a different purpose than the endpoints served by canarie-api + (monitoring status). This endpoint is meant to be polled by the node registry scripts + (https://github.com/DACCS-Climate/DACCS-node-registry) to provide information about what services are meant to be + available without having to poll other endpoints directly. + +- A new endpoint `/version` is added that provides a string containing the current version number of the stack + (e.g. "1.26.0"). This endpoint is meant to be polled by the node registry scripts + (https://github.com/DACCS-Climate/DACCS-node-registry). + [1.26.6](https://github.com/bird-house/birdhouse-deploy/tree/1.26.6) (2023-06-16) ------------------------------------------------------------------------------------------------------------------ @@ -96,16 +107,6 @@ * Add Misha to creators * Add birdhouse community -- A new endpoint `/services` is added that provides a JSON string describing each of the user facing services currently - enabled on the stack. This is a static string and serves a different purpose than the endpoints served by canarie-api - (monitoring status). This endpoint is meant to be polled by the node registry scripts - (https://github.com/DACCS-Climate/DACCS-node-registry) to provide information about what services are meant to be - available without having to poll other endpoints directly. - -- A new endpoint `/version` is added that provides a string containing the current version number of the stack - (e.g. "1.26.0"). This endpoint is meant to be polled by the node registry scripts - (https://github.com/DACCS-Climate/DACCS-node-registry). - - Licence: update copyright line with year and ownership From 2ded4f60b7eedd9dadfe4fe82960606f7d668056 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 22 Jun 2023 17:22:08 -0400 Subject: [PATCH 38/39] =?UTF-8?q?Bump=20version:=201.26.7=20=E2=86=92=201.?= =?UTF-8?q?26.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 6 +++--- CHANGES.md | 5 +++++ Makefile | 2 +- README.rst | 8 ++++---- RELEASE.txt | 2 +- .../config/canarie-api/docker_configuration.py.template | 8 ++++---- docs/source/conf.py | 4 ++-- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 45e0c4cc5..f18adea09 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.26.7 +current_version = 1.26.8 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-06-19T20:23:19Z +search = {current_version} 2023-06-22T21:22:08Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-06-19T20:23:19Z +values = 2023-06-22T21:22:08Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index 29e03e504..12d7982c2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,11 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + +[1.26.8](https://github.com/bird-house/birdhouse-deploy/tree/1.26.8) (2023-06-22) +------------------------------------------------------------------------------------------------------------------ + ## Fixes - Tests: some tests fail to run when `CWD` is not `COMPOSE_DIR` diff --git a/Makefile b/Makefile index e50f20190..aa06e71b1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.26.7 +override APP_VERSION := 1.26.8 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index 175458a38..cc52ec215 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.7.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.8.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.7...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.8...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.7-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.8-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.7 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.8 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index f96dae4e8..a980c47c2 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.26.7 2023-06-19T20:23:19Z +1.26.8 2023-06-22T21:22:08Z diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index ba2bcb583..ba1e77c7e 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.7', - 'releaseTime': '2023-06-19T20:23:19Z', + 'version': '1.26.8', + 'releaseTime': '2023-06-22T21:22:08Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.7', - 'releaseTime': '2023-06-19T20:23:19Z', + 'version': '1.26.8', + 'releaseTime': '2023-06-22T21:22:08Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/docs/source/conf.py b/docs/source/conf.py index 8e861e52e..56f9de6ec 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.26.7' +version = '1.26.8' # The full version, including alpha/beta/rc tags. -release = '1.26.7' +release = '1.26.8' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 7b4041f37e83c3e82535f608006305e7ebe08f37 Mon Sep 17 00:00:00 2001 From: mishaschwartz Date: Tue, 4 Jul 2023 08:44:11 -0400 Subject: [PATCH 39/39] =?UTF-8?q?Bump=20version:=201.26.8=20=E2=86=92=201.?= =?UTF-8?q?26.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 6 +++--- CHANGES.md | 5 +++++ Makefile | 2 +- README.rst | 8 ++++---- RELEASE.txt | 2 +- .../config/canarie-api/docker_configuration.py.template | 8 ++++---- docs/source/conf.py | 4 ++-- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f18adea09..fa61936d3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.26.8 +current_version = 1.26.9 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-06-22T21:22:08Z +search = {current_version} 2023-07-04T12:44:10Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-06-22T21:22:08Z +values = 2023-07-04T12:44:10Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index 34bc86954..ffe17ac38 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,11 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + +[1.26.9](https://github.com/bird-house/birdhouse-deploy/tree/1.26.9) (2023-07-04) +------------------------------------------------------------------------------------------------------------------ + ## Fixes - Fix Cowbird's `sync_permissions` config which used invalid Magpie service types. diff --git a/Makefile b/Makefile index aa06e71b1..91c801f12 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.26.8 +override APP_VERSION := 1.26.9 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index cc52ec215..5b030beb2 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.8.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.9.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.8...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.9...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.8-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.9-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.8 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.9 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index a980c47c2..5fdb6e828 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.26.8 2023-06-22T21:22:08Z +1.26.9 2023-07-04T12:44:10Z diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index ba1e77c7e..cf2da9d4d 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.8', - 'releaseTime': '2023-06-22T21:22:08Z', + 'version': '1.26.9', + 'releaseTime': '2023-07-04T12:44:10Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.26.8', - 'releaseTime': '2023-06-22T21:22:08Z', + 'version': '1.26.9', + 'releaseTime': '2023-07-04T12:44:10Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/docs/source/conf.py b/docs/source/conf.py index 56f9de6ec..5c9271630 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.26.8' +version = '1.26.9' # The full version, including alpha/beta/rc tags. -release = '1.26.8' +release = '1.26.9' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.