bird-house · mishaschwartz · Aug 10, 2023 · Apr 25, 2023 · May 29, 2023 · May 29, 2023
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.28.0
+current_version = 1.29.0
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2023-08-10T19:23:14Z
+search = {current_version} 2023-08-10T19:38:10Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2023-08-10T19:23:14Z
+values = 2023-08-10T19:38:10Z
 
 [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

@@ -17,6 +17,20 @@
 
 [//]: # (list changes here, using '-' for each new entry, remove this when items are added)
 
+[1.29.0](https://github.com/bird-house/birdhouse-deploy/tree/1.29.0) (2023-08-10)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Do not expose additional ports:
+ - Docker compose no longer exposes any container ports outside the default network except for ports 80 and 443 from 
+ the proxy container. This ensures that ports that are not intended for external access are not exposed to the wider 
+ internet even if firewall rules are not set correctly.
+ - Note that if the `monitoring` component is used then port 9100 will be exposed from the `node-exporter` container.
+ This is because this container must be run on the host machine's network and unfortunately there is no known
+ workaround that would not require this port to be exposed on the host machine.
+ - Fixes https://github.com/bird-house/birdhouse-deploy/issues/222
+
+
 [1.28.0](https://github.com/bird-house/birdhouse-deploy/tree/1.28.0) (2023-08-10)
 ------------------------------------------------------------------------------------------------------------------
 
@@ -212,6 +226,7 @@
 ------------------------------------------------------------------------------------------------------------------
 
 ## Changes
+
 - Update Zenodo config
  * Add Misha to creators
  * Add birdhouse community

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL := bash
 override APP_NAME := birdhouse-deploy
-override APP_VERSION := 1.28.0
+override APP_VERSION := 1.29.0
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

@@ -14,13 +14,13 @@ for a full-fledged production platform.
  * - releases
  - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.28.0.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.29.0.svg
  :alt: Commits since latest release
- :target: https://github.com/bird-house/birdhouse-deploy/compare/1.28.0...master
+ :target: https://github.com/bird-house/birdhouse-deploy/compare/1.29.0...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.28.0-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.29.0-blue.svg?style=flat
  :alt: Latest Tag
- :target: https://github.com/bird-house/birdhouse-deploy/tree/1.28.0
+ :target: https://github.com/bird-house/birdhouse-deploy/tree/1.29.0
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
  :alt: ReadTheDocs Build Status (latest version)

@@ -1 +1 @@
-1.28.0 2023-08-10T19:23:14Z
+1.29.0 2023-08-10T19:38:10Z
@@ -294,17 +294,14 @@ Prometheus stack is used:
 Usage
 -----
 
-- Grafana to view metric graphs: http://PAVICS_FQDN:3001/d/pf6xQMWGz/docker-and-system-monitoring
-- Prometheus alert rules: http://PAVICS_FQDN:9090/rules
-- AlertManager to manage alerts: http://PAVICS_FQDN:9093
-
-The paths above are purposely not behind the proxy to not expose them publicly,
-assuming only ports 80 and 443 are publicly exposed on the internet. All other
-ports are not exposed.
-
-Only Grafana has authentication, Prometheus alert rules and AlertManager have
-no authentication at all so had they been behind the proxy, anyone will be
-able to access them.
+- Grafana to view metric graphs: https://PAVICS_FQDN/grafana/d/pf6xQMWGz/docker-and-system-monitoring
+- Prometheus alert rules: https://PAVICS_FQDN/prometheus/rules
+- AlertManager to manage alerts: https://PAVICS_FQDN/alertmanager
+
+The paths above are by default only accessible to a user logged in to magpie as an administrator.
+These routes provide sensitive information about the birdhouse-deploy software stack and the machine
+that it is running on. It is highly discouraged to make these routes available to anyone who is not
+an administrator.
 
 
 How to Enable the Component

@@ -3,3 +3,5 @@ grafana_datasources.yml
 grafana_dashboards.yml
 alertmanager.yml
 prometheus.rules
+config/magpie/config.yml
+config/proxy/conf.extra-service.d/monitoring.conf
@@ -0,0 +1,54 @@
+providers:
+ grafana:
+ # below URL is only used to fill in the required location in Magpie
+ # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+ url: http://proxy:80
+ title: Grafana
+ public: true
+ c4i: false
+ type: api
+ sync_type: api
+ prometheus:
+ # below URL is only used to fill in the required location in Magpie
+ # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+ url: http://proxy:80
+ title: Prometheus
+ public: true
+ c4i: false
+ type: api
+ sync_type: api
+ alertmanager:
+ # below URL is only used to fill in the required location in Magpie
+ # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+ url: http://proxy:80
+ title: AlertManager
+ public: true
+ c4i: false
+ type: api
+ sync_type: api
+
+permissions:
+ - service: grafana
+ permission: read
+ group: administrators
+ action: create
+ - service: grafana
+ permission: write
+ group: administrators
+ action: create
+ - service: prometheus
+ permission: read
+ group: administrators
+ action: create
+ - service: prometheus
+ permission: write
+ group: administrators
+ action: create
+ - service: alertmanager
+ permission: read
+ group: administrators
+ action: create
+ - service: alertmanager
+ permission: write
+ group: administrators
+ action: create
@@ -0,0 +1,7 @@
+version: "3.4"
+
+services:
+ magpie:
+ volumes:
+ - ./components/monitoring/config/magpie/config.yml:${MAGPIE_PERMISSIONS_CONFIG_PATH}/monitoring.yml:ro
+ - ./components/monitoring/config/magpie/config.yml:${MAGPIE_PROVIDERS_CONFIG_PATH}/monitoring.yml:ro
@@ -0,0 +1,57 @@
+
+ location /grafana {
+ auth_request /secure-grafana-auth;
+ auth_request_set $auth_status $upstream_status;
+ proxy_pass http://grafana:3000;
+ proxy_set_header Host $host;
+ }
+
+ location /prometheus {
+ auth_request /secure-prometheus-auth;
+ auth_request_set $auth_status $upstream_status;
+ proxy_pass http://prometheus:9090;
+ proxy_set_header Host $host;
+ }
+
+ location /alertmanager {
+ auth_request /secure-alertmanager-auth;
+ auth_request_set $auth_status $upstream_status;
+ proxy_pass http://alertmanager:9093;
+ proxy_set_header Host $host;
+ }
+
+ location = /secure-grafana-auth {
+ internal;
+ proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/grafana$request_uri;
+ proxy_pass_request_body off;
+ proxy_set_header Host $host;
+ proxy_set_header Content-Length "";
+ proxy_set_header X-Original-URI $request_uri;
+ proxy_set_header X-Forwarded-Proto $real_scheme;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Host $host:$server_port;
+ }
+
+ location = /secure-prometheus-auth {
+ internal;
+ proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/prometheus$request_uri;
+ proxy_pass_request_body off;
+ proxy_set_header Host $host;
+ proxy_set_header Content-Length "";
+ proxy_set_header X-Original-URI $request_uri;
+ proxy_set_header X-Forwarded-Proto $real_scheme;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Host $host:$server_port;
+ }
+
+ location = /secure-alertmanager-auth {
+ internal;
+ proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/alertmanager$request_uri;
+ proxy_pass_request_body off;
+ proxy_set_header Host $host;
+ proxy_set_header Content-Length "";
+ proxy_set_header X-Original-URI $request_uri;
+ proxy_set_header X-Forwarded-Proto $real_scheme;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Host $host:$server_port;
+ }
@@ -0,0 +1,6 @@
+version: "3.4"
+
+services:
+ proxy:
+ volumes:
+ - ./components/monitoring/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/monitoring:ro
@@ -11,8 +11,6 @@ services:
  - /var/run:/var/run:ro
  - /sys:/sys:ro
  - /var/lib/docker:/var/lib/docker:ro
- ports:
- - 9999:8080
  devices:
  - /dev/kmsg
  restart: always
@@ -38,8 +36,6 @@ services:
  - ./components/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
  - ./components/monitoring/prometheus.rules:/etc/prometheus/prometheus.rules:ro
  - prometheus_persistence:/prometheus:rw
- ports:
- - 9090:9090
  command:
  # restore original CMD from image
  - --config.file=/etc/prometheus/prometheus.yml
@@ -49,7 +45,7 @@ services:
  # https://prometheus.io/docs/prometheus/latest/storage/
  - --storage.tsdb.retention.time=90d
  # wrong default was http://container-hash:9090/
- - --web.external-url=http://${PAVICS_FQDN}:9090/
+ - --web.external-url=https://${PAVICS_FQDN_PUBLIC}/prometheus/
  restart: always
 
  # https://grafana.com/docs/grafana/latest/installation/docker/
@@ -65,8 +61,9 @@ services:
  - grafana_persistence:/var/lib/grafana:rw
  environment:
  GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
- ports:
- - 3001:3000
+ GF_SERVER_ROOT_URL: https://${PAVICS_FQDN_PUBLIC}/grafana
+ GF_SERVER_SERVE_FROM_SUB_PATH: 'true'
+ GF_SERVER_DOMAIN: ${PAVICS_FQDN_PUBLIC}
  restart: always
 
  # https://github.com/prometheus/alertmanager
@@ -86,9 +83,7 @@ services:
  # enable debug logging
  - --log.level=debug
  # wrong default was http://container-hash:9093/
- - --web.external-url=http://${PAVICS_FQDN}:9093/
- ports:
- - 9093:9093
+ - --web.external-url=https://${PAVICS_FQDN_PUBLIC}/alertmanager
  restart: always
 
 volumes:

@@ -6,6 +6,6 @@ datasources:
  type: prometheus
  access: proxy
  uid: local_pavics_prometheus
- url: http://${PAVICS_FQDN}:9090
+ url: http://prometheus:9090/prometheus
  isDefault: true
  editable: false
@@ -1,5 +1,5 @@
 # https://prometheus.io/docs/prometheus/latest/configuration/configuration/
-# http://PAVICS_FQDN:9090/config
+# http://PAVICS_FQDN/prometheus/config
 global:
  scrape_interval: 60s
  evaluation_interval: 30s
@@ -10,8 +10,10 @@ scrape_configs:
  honor_labels: true
  static_configs:
  - targets:
- - ${PAVICS_FQDN}:9999
+ - cadvisor:8080
 
+# Node exporter is required to run on the host network so it is not accessible through the docker network.
+# It is only accessible via the host network which can be accessed using the PAVICS_FQDN variable.
 - job_name: node-exporter
  honor_labels: true
  static_configs:
@@ -26,4 +28,4 @@ alerting:
  - scheme: http
  static_configs:
  - targets:
- - "${PAVICS_FQDN}:9093"
+ - alertmanager:9093
@@ -4,7 +4,7 @@
 providers:
  # definition of Weaver service
  ${WEAVER_MANAGER_NAME}:
- url: http://${PAVICS_FQDN}:4001
+ url: http://weaver:4001
  title: Weaver (${WEAVER_CONFIG})
  public: true
  c4i: false
@@ -57,7 +57,7 @@ providers:
 
  # FIXME: remove when https://github.com/Ouranosinc/Magpie/issues/360 implemented, see 'default.env'
  ${WEAVER_WPS_NAME}:
- url: http://${PAVICS_FQDN}:4001/wps
+ url: http://weaver:4001/wps
  title: Weaver (WPS)
  public: true
  c4i: false

@@ -17,8 +17,6 @@ services:
  # This is needed because simply adding 'depends_on' only ensures that containers are 'running', but startup of
  # WPS applications themselves are not necessarily completed. Successful HTTP responses ensure they are 'ready'.
  image: pavics/weaver:${WEAVER_VERSION}-manager
- ports:
- - "4001:4001"
  environment:
  HOSTNAME: ${PAVICS_FQDN}
  FORWARDED_ALLOW_IPS: "*"

@@ -109,8 +109,8 @@ SERVICES = {
  # NOTE:
  # Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
  # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
- 'version': '1.28.0',
- 'releaseTime': '2023-08-10T19:23:14Z',
+ 'version': '1.29.0',
+ 'releaseTime': '2023-08-10T19:38:10Z',
  'institution': 'Ouranos',
  'researchSubject': 'Climatology',
  'supportEmail': '${SUPPORT_EMAIL}',
@@ -142,8 +142,8 @@ PLATFORMS = {
  # NOTE:
  # Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
  # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
- 'version': '1.28.0',
- 'releaseTime': '2023-08-10T19:23:14Z',
+ 'version': '1.29.0',
+ 'releaseTime': '2023-08-10T19:38:10Z',
  'institution': 'Ouranos',
  'researchSubject': 'Climatology',
  'supportEmail': '${SUPPORT_EMAIL}',

@@ -67,8 +67,7 @@ SERVICES['slicer'] = {
  'monitoring': {
  'Finch': {
  'request': {
- # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222)
- 'url': 'http://${PAVICS_FQDN}:8095/?service=WPS&version=1.0.0&request=GetCapabilities'
+ 'url': 'http://finch:5000/?service=WPS&version=1.0.0&request=GetCapabilities'
  }
  },
  }
@@ -106,8 +105,7 @@ SERVICES['Finch'] = {
  'monitoring': {
  'Finch': {
  'request': {
- # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222)
- 'url': 'http://${PAVICS_FQDN}:8095/wps?service=WPS&version=1.0.0&request=GetCapabilities'
+ 'url': 'http://finch:5000/wps?service=WPS&version=1.0.0&request=GetCapabilities'
  }
  },
  },

@@ -1,6 +1,6 @@
 providers:
  finch:
- url: http://${PAVICS_FQDN}:8095/wps
+ url: http://finch:5000/wps
  title: Finch
  public: true
  c4i: false