diff --git a/analytics-datastore-clickhouse/docker-compose.yml b/analytics-datastore-clickhouse/docker-compose.yml index 1885c440..cb5f39a3 100644 --- a/analytics-datastore-clickhouse/docker-compose.yml +++ b/analytics-datastore-clickhouse/docker-compose.yml @@ -25,6 +25,7 @@ services: datalake: + volumes: clickhouse-data: diff --git a/database-postgres/package-metadata.json b/database-postgres/package-metadata.json index 10beda95..f31c9f0c 100644 --- a/database-postgres/package-metadata.json +++ b/database-postgres/package-metadata.json @@ -9,7 +9,7 @@ "REPMGR_PRIMARY_HOST": "postgres-1", "REPMGR_PARTNER_NODES": "postgres-1", "REPMGR_PASSWORD": "instant101", - "POSTGRES_IMAGE": "bitnami/postgresql-repmgr:14", + "POSTGRES_IMAGE": "bitnami/postgresql-repmgr:14@sha256:bdf1e4903710c4e0b465664b886d4556897e1b18c07d7c513a4fc1ceba929e02", "POSTGRES_1_PLACEMENT": "node-1", "POSTGRES_2_PLACEMENT": "node-2", "POSTGRES_3_PLACEMENT": "node-3", diff --git a/datalake/docker-compose.yml b/datalake/docker-compose.yml index 0c1f2cb2..b2f759c3 100644 --- a/datalake/docker-compose.yml +++ b/datalake/docker-compose.yml @@ -24,16 +24,31 @@ services: labels: - traefik.enable=true - traefik.docker.network=reverse-proxy-traefik_public - - traefik.http.routers.minio.rule=${DOMAIN_NAME_HOST_TRAEFIK} && PathPrefix(`/minio`) - - traefik.http.services.minio.loadbalancer.server.port=9001 - - traefik.http.middlewares.minio-stripprefix.stripprefix.prefixes=/minio - - traefik.http.routers.minio.middlewares=minio-stripprefix + - traefik.http.routers.minio-console.rule=Host(`${DOMAIN_NAME_HOST_TRAEFIK}`) && PathPrefix(`/minio-console`) + - traefik.http.routers.minio-console.priority=100 + - traefik.http.routers.minio-console.service=minio-console-service + - traefik.http.services.minio-console-service.loadbalancer.server.port=9001 + - traefik.http.middlewares.minio-console-stripprefix.stripprefix.prefixes=/minio-console/ + - traefik.http.routers.minio-console.middlewares=minio-console-stripprefix + - traefik.http.routers.minio-console.tls=${TLS} + + - traefik.http.routers.minio-api.rule=Host(`${DOMAIN_NAME_HOST_TRAEFIK}`) && PathPrefix(`/minio-api`) + - traefik.http.routers.minio-api.priority=100 + - traefik.http.routers.minio-api.service=minio-api-service + - traefik.http.services.minio-api-service.loadbalancer.server.port=9090 + - traefik.http.middlewares.minio-api-stripprefix.stripprefix.prefixes=/minio-api/ + - traefik.http.routers.minio-api.middlewares=minio-api-stripprefix + - traefik.http.routers.minio-api.tls=${TLS} networks: public: + traefik: networks: public: name: minio_public external: true + traefik: + name: reverse-proxy-traefik_public + external: true volumes: minio-01-data1: diff --git a/documentation/SUMMARY.md b/documentation/SUMMARY.md index cff0e3ab..f9f161ae 100644 --- a/documentation/SUMMARY.md +++ b/documentation/SUMMARY.md @@ -52,6 +52,10 @@ - [Reverse Proxy Nginx](packages/reverse-proxy-nginx/README.md) - [Local Development](packages/reverse-proxy-nginx/local-development.md) - [Environment Variables](packages/reverse-proxy-nginx/environment-variables.md) + - [OpenFn](packages/openfn/README.md) + - [Environment Variables](packages/openfn/environment-variables.md) + - [Reverse Proxy Traefik](packages/reverse-proxy-traefik/README.md) + - [Environment Variables](packages/reverse-proxy-traefik/environment-variables.md) - [🗒️ Cheat sheet](cheat-sheet.md) - [Architecture](architecture.md) - [Guides](guides/README.md) diff --git a/documentation/packages/openfn/environment-variables.md b/documentation/packages/openfn/environment-variables.md index 7ce16508..ef9a6c00 100644 --- a/documentation/packages/openfn/environment-variables.md +++ b/documentation/packages/openfn/environment-variables.md @@ -5,180 +5,164 @@ Variable Name Description - Type - Relevance - Required Default - DATABASE_URL + OPENFN_DATABASE_URL The URL of the PostgreSQL database - - - - + postgresql://openfn:instant101@postgres-1:5432/lightning_dev - DISABLE_DB_SSL + OPENFN_DISABLE_DB_SSL Whether to disable SSL for the database connection - - - - + true - IS_RESETTABLE_DEMO + OPENFN_IS_RESETTABLE_DEMO Whether the application is running in resettable demo mode - - - - + true - LISTEN_ADDRESS + OPENFN_LISTEN_ADDRESS The IP address to listen on - - - - + 0.0.0.0 - LOG_LEVEL + OPENFN_LOG_LEVEL The log level for the application - - - - + debug - ORIGINS + OPENFN_ORIGINS The allowed origins for CORS - - - - + http://localhost:4000 - PRIMARY_ENCRYPTION_KEY + OPENFN_PRIMARY_ENCRYPTION_KEY The primary encryption key - - - - + KLu/IoZuaf+baDECd8wG4Z6auwNe6VAmwh9N8lWdJ1A= - SECRET_KEY_BASE + OPENFN_SECRET_KEY_BASE The secret key base - - - - + jGDxZj2O+Qzegm5wcZ940RfWO4D6RyU8thNCr5BUpHNwa7UNV52M1/Sn+7RxiP+f - WORKER_RUNS_PRIVATE_KEY + OPENFN_WORKER_RUNS_PRIVATE_KEY The private key for worker runs - - - - + LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRREVtR3drUW5pT0hqVCsKMnkyRHFvRUhyT3dLZFI2RW9RWG9DeDE4MytXZ3hNcGthTFZyOFViYVVVQWNISGgzUFp2Z2UwcEIzTWlCWWR5Kwp1ajM1am5uK2JIdk9OZGRldWxOUUdpczdrVFFHRU1nTSs0Njhldm5RS0h6R29DRUhabDlZV0s0MUd5SEZCZXppCnJiOGx2T1A1NEtSTS90aE5pVGtHaUIvTGFLMldLcTh0VmtoSHBvaFE3OGIyR21vNzNmcWtuSGZNWnc0ZE43d1MKdldOamZIN3QwSmhUdW9mTXludUxSWmdFYUhmTDlnbytzZ0thc0ZUTmVvdEZIQkYxQTJjUDJCakwzaUxad0hmdQozTzEwZzg0aGZlTzJqTWlsZlladHNDdmxDTE1EZWNrdFJGWFl6V0dWc25FcFNiOStjcWJWUXRvdEU4QklON09GClRmaEx2MG9uQWdNQkFBRUNnZ0VBV3dmZyt5RTBSVXBEYThiOVdqdzNKdUN4STE1NzFSbmliRUhKVTZzdzNyS0EKck9HM0w5WTI0cHhBdlVPSm5GMFFzbThrUVQ4RU1MU3B6RDdjdDVON2RZMngvaGY4TThhL0VSWXM4cFlYcXI5Vwpnbnh3NldGZ0R6elFHZ0RIaW0raXNudk5ucFdEbTRGVTRObG02d2g5MzVSZlA2KzVaSjJucEJpZjhFWDJLdE9rCklOSHRVbFcwNFlXeDEwS0pIWWhYNFlydXVjL3MraXBORzBCSDZEdlJaQzQxSWw0N1luaTg1OERaL0FaeVNZN1kKWTlTamNKQ0QvUHBENTlNQjlSanJDQjhweDBjWGlsVXBVZUJSYndGalVwbWZuVmhIa1hiYlM1U0hXWWM4K3pLRQp2ajFqSEpxc2UyR0hxK2lHL1V3NTZvcHNyM2x3dHBRUXpVcEJGblhMMFFLQmdRRDM5bkV3L1NNVGhCallSd1JGCkY2a2xOYmltU2RGOVozQlZleXhrT0dUeU5NSCtYckhsQjFpOXBRRHdtMit3V2RvcWg1ZFRFbEU5K1crZ0FhN0YKbXlWc2xPTW4wdnZ2cXY2Wkp5SDRtNTVKU0lWSzBzRjRQOTRMYkpNSStHUW5VNnRha3Y0V0FSMkpXaURabGxPdAp3R01EQWZqRVIrSEFZeUJDKzNDL25MNHF5d0tCZ1FESzk3NERtV0c4VDMzNHBiUFVEYnpDbG9oTlQ2UldxMXVwCmJSWng4ZGpzZU0vQ09kZnBUcmJuMnk5dVc3Q1pBNFVPQ2s4REcxZ3ZENVVDYlpEUVdMaUp5RzZGdG5OdGgvaU8KT1dJM0UyczZOS0VMMU1NVzh5QWZwNzV4Ung5cnNaQzI2UEtqQ0pWL2lTVjcyNlQ1ZTFzRG5sZUtBb0JFZnlDRgpvbEhhMmhybWxRS0JnUURHT1YyOWd1K1NmMng1SVRTWm8xT1ZxbitGZDhlZno1d3V5YnZ3Rm1Fa2V1YUdXZDh1CnJ4UFM3MkJ6K0Y1dUJUWngvMWtLa0w4Zm94TUlQN0FleW1zOWhUeWVybnkyMk9TVlBJSmN3dExqMUxTeDN3L0kKK0kyaVpsYVl1akVlZXpXbHY1S2R0cUNORjk3Zzh0ck1NTnMySVZKa1h1NXFwUk82V0ZXRzZGL2h4d0tCZ0hnNApHYUpFSFhIT204ekZTU2lYSW5FWGZKQmVWZmJIOUxqNzFrbVRlR3RJZTdhTlVHZnVxY1BYUGRiZUZGSHRsY2ZsCkx6dWwzS3V6VFExdEhGTnIyWkl5MTlQM1o1TSs4R2c5Y1FFeVRWYmlpV2xha2x0cmttRnRtQTI4bE0zVEZPWmkKUUNWMUZpZStjaWRVeC9qRnFma1F0c1VXQ2llSUxSazZOY1d0WGpXcEFvR0JBTGN6Y210VGlUUEFvWnk0MFV1QQpTOXpUd3RsamhmUWJEVTVjb21EcnlKcnFRU0VOdmQ2VW5HdW0zYVNnNk13dDc0NGxidDAyMC9mSGI0WTJkTGhMCmx4YWJ5b1dQUElRRUpLL1NNOGtURFEvYTRyME5tZzhuV3h5bGFLcHQ5WUhmZ2NYMkYzSzUrc0VSUGNFcVZlWFMKdWZkYXdYQVlFampZK3V2UHZ2YzU3RU1aCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K + + + OPENFN_WORKER_SECRET + The secret key for the worker + secret_here POSTGRES_USER The username for the PostgreSQL database - - - - + postgres POSTGRES_SERVICE The service name for the PostgreSQL database - - - - + postgres-1 POSTGRES_DATABASE The name of the PostgreSQL database - - - - + postgres POSTGRES_PASSWORD The password for the PostgreSQL database - - - - + instant101 POSTGRES_PORT The port number for the PostgreSQL database - - - - + 5432 - OpenFn_POSTGRESQL_DB + OPENFN_POSTGRESQL_DB The name of the OpenFn PostgreSQL database - - - - + lightning_dev - OpenFn_POSTGRESQL_USERNAME + OPENFN_POSTGRESQL_USERNAME The username for the OpenFn PostgreSQL database - - - - + openfn - OpenFn_POSTGRESQL_PASSWORD + OPENFN_POSTGRESQL_PASSWORD The password for the OpenFn PostgreSQL database - - - - + instant101 - WORKER_LIGHTNING_PUBLIC_KEY + OPENFN_WORKER_LIGHTNING_PUBLIC_KEY The public key for the worker lightning - - - - + LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4SmhzSkVKNGpoNDAvdHN0ZzZxQgpCNnpzQ25VZWhLRUY2QXNkZk4vbG9NVEtaR2kxYS9GRzJsRkFIQng0ZHoyYjRIdEtRZHpJZ1dIY3ZybzkrWTU1Ci9teDd6alhYWHJwVFVCb3JPNUUwQmhESURQdU92SHI1MENoOHhxQWhCMlpmV0ZpdU5Sc2h4UVhzNHEyL0piemoKK2VDa1RQN1lUWWs1Qm9nZnkyaXRsaXF2TFZaSVI2YUlVTy9HOWhwcU85MzZwSngzekdjT0hUZThFcjFqWTN4Kwo3ZENZVTdxSHpNcDdpMFdZQkdoM3kvWUtQcklDbXJCVXpYcUxSUndSZFFObkQ5Z1l5OTRpMmNCMzd0enRkSVBPCklYM2p0b3pJcFgyR2JiQXI1UWl6QTNuSkxVUlYyTTFobGJKeEtVbS9mbkttMVVMYUxSUEFTRGV6aFUzNFM3OUsKSndJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg - WORKER_SECRET - The secret key for the worker - - - - - - - OpenFn_IMAGE + OPENFN_IMAGE The image name for OpenFn - - - - + openfn/lightning:v2.9.5 - OpenFn_WORKER_IMAGE + OPENFN_WORKER_IMAGE The image name for OpenFn worker - - - - + openfn/ws-worker:latest + + + OPENFN_KAFKA_TRIGGERS_ENABLED + Whether Kafka triggers are enabled + true + + + OPENFN_API_KEY + The API key for OpenFn + apiKey + + + OPENFN_ENDPOINT + The endpoint for OpenFn + http://localhost:4000 + + + OPENFN_DOCKER_WEB_CPUS + The number of CPUs allocated to the web container + 2 + + + OPENFN_DOCKER_WEB_MEMORY + The amount of memory allocated to the web container + 4G + + + OPENFN_DOCKER_WORKER_CPUS + The number of CPUs allocated to the worker container + 2 + + + OPENFN_DOCKER_WORKER_MEMORY + The amount of memory allocated to the worker container + 4G + + + FHIR_SERVER_BASE_URL + The base URL for the FHIR server + http://openhim-core:5001 + + + FHIR_SERVER_USERNAME + The username for the FHIR server + openfn_client + + + FHIR_SERVER_PASSWORD + The password for the FHIR server + openfn_client_password diff --git a/documentation/packages/reverse-proxy-traefik/README.md b/documentation/packages/reverse-proxy-traefik/README.md index 872651f0..08c04c1d 100644 --- a/documentation/packages/reverse-proxy-traefik/README.md +++ b/documentation/packages/reverse-proxy-traefik/README.md @@ -92,14 +92,30 @@ Set the following environment variables in the package-metadata.json in the "./d Set the following environment variables in the package-metadata.json in the "monitoring" directory ```bash - "environmentVariables": { # Other Configurations ... - "MINIO_BROWSER_REDIRECT_URL": "https://domain/minio/" + "MINIO_BROWSER_REDIRECT_URL": "https://domain/minio-console/" } +``` + +### MinIO Configuration + +The MinIO server is configured to run with the following port settings: + +- **API Port**: 9090 +- **Console Port**: 9001 + +Ensure that your Traefik configuration reflects these ports to properly route traffic to the MinIO services. The API can be accessed at `https:///minio` and the Console at `https:///minio-console`. + +Update your Traefik labels in the `docker-compose.yml` to match these settings: +```yaml +# API Configuration +- traefik.http.services.minio.loadbalancer.server.port=9090 +# Console Configuration +- traefik.http.services.minio-console.loadbalancer.server.port=9001 ``` ### Enabling Grafana diff --git a/monitoring/package-metadata.json b/monitoring/package-metadata.json index 31691759..52d582f9 100644 --- a/monitoring/package-metadata.json +++ b/monitoring/package-metadata.json @@ -29,7 +29,6 @@ "MO_SECURITY_ADMIN_PASSWORD": "dev_password_only", "MO_RETENTION_TIME": "15d", "GF_SERVER_SERVE_FROM_SUB_PATH": "false", - "MINIO_BROWSER_REDIRECT_URL": "", "DOCKER_SOCK_FOLDER": "/var/run/docker.sock", "DOCKER_LIB_FOLDER": "/var/lib/docker/" } diff --git a/reverse-proxy-traefik/docker-compose.yml b/reverse-proxy-traefik/docker-compose.yml index ebd83ea3..09918f96 100644 --- a/reverse-proxy-traefik/docker-compose.yml +++ b/reverse-proxy-traefik/docker-compose.yml @@ -49,6 +49,10 @@ services: - traefik.http.middlewares.to-https.redirectscheme.permanent=${REDIRECT_TO_HTTPS} - traefik.http.middlewares.auth.basicauth.users=${USERNAME}:${PASSWORD} + - "traefik.http.middlewares.bigfiles.buffering.maxRequestBodyBytes=100000000" + - "traefik.http.service.traefik.loadbalancer.server.forwardingTimeouts.dialTimeout=120s" + - "traefik.http.service.traefik.loadbalancer.server.forwardingTimeouts.responseHeaderTimeout=120s" + placement: max_replicas_per_node: 1 constraints: