From 3e472d292dc8c1cf12fc987cae9b797c469a1fbe Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Wed, 23 Nov 2022 12:09:48 +0200 Subject: [PATCH 01/11] added more jempi services base compose files PLAT-620 --- .../docker-compose.async-receiver.yml | 12 ++++++++++++ .../docker-compose.em-calculator.yml | 12 ++++++++++++ .../docker-compose.jempi-controller.yml | 17 +++++++++++++++++ client-registry-jempi/docker-compose.linker.yml | 17 +++++++++++++++++ .../docker-compose.pre-processor.yml | 12 ++++++++++++ .../docker-compose.sync-receiver.yml | 17 +++++++++++++++++ client-registry-jempi/package-metadata.json | 14 +++++++++++++- 7 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 client-registry-jempi/docker-compose.async-receiver.yml create mode 100644 client-registry-jempi/docker-compose.em-calculator.yml create mode 100644 client-registry-jempi/docker-compose.jempi-controller.yml create mode 100644 client-registry-jempi/docker-compose.linker.yml create mode 100644 client-registry-jempi/docker-compose.pre-processor.yml create mode 100644 client-registry-jempi/docker-compose.sync-receiver.yml diff --git a/client-registry-jempi/docker-compose.async-receiver.yml b/client-registry-jempi/docker-compose.async-receiver.yml new file mode 100644 index 00000000..393ff0be --- /dev/null +++ b/client-registry-jempi/docker-compose.async-receiver.yml @@ -0,0 +1,12 @@ +version: '3.9' + +services: + jempi-async-receiver: + image: jembi/jempi-async-receiver + deploy: + replicas: 1 + resources: + limits: + memory: ${ASYNC_RECEIVER_MEMORY_LIMIT} + reservations: + memory: ${ASYNC_RECEIVER_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.em-calculator.yml b/client-registry-jempi/docker-compose.em-calculator.yml new file mode 100644 index 00000000..bd00e0a1 --- /dev/null +++ b/client-registry-jempi/docker-compose.em-calculator.yml @@ -0,0 +1,12 @@ +version: '3.9' + +services: + jempi-em-calculator: + image: jembi/jempi-em-calculator + deploy: + replicas: 1 + resources: + limits: + memory: ${EM_CALCULATOR_MEMORY_LIMIT} + reservations: + memory: ${EM_CALCULATOR_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.jempi-controller.yml b/client-registry-jempi/docker-compose.jempi-controller.yml new file mode 100644 index 00000000..f22bb369 --- /dev/null +++ b/client-registry-jempi/docker-compose.jempi-controller.yml @@ -0,0 +1,17 @@ +version: '3.9' + +services: + jempi-controller: + image: jembi/jempi-controller + ports: + - published: 50020 + target: 50000 + protocol: tcp + mode: host + deploy: + replicas: 1 + resources: + limits: + memory: ${CONTROLLER_MEMORY_LIMIT} + reservations: + memory: ${CONTROLLER_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.linker.yml b/client-registry-jempi/docker-compose.linker.yml new file mode 100644 index 00000000..47bdae81 --- /dev/null +++ b/client-registry-jempi/docker-compose.linker.yml @@ -0,0 +1,17 @@ +version: '3.9' + +services: + jempi-linker: + image: jembi/jempi-linker + ports: + - published: 50010 + target: 50000 + protocol: tcp + mode: host + deploy: + replicas: 1 + resources: + limits: + memory: ${LINKER_MEMORY_LIMIT} + reservations: + memory: ${LINKER_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.pre-processor.yml b/client-registry-jempi/docker-compose.pre-processor.yml new file mode 100644 index 00000000..a924c3aa --- /dev/null +++ b/client-registry-jempi/docker-compose.pre-processor.yml @@ -0,0 +1,12 @@ +version: '3.9' + +services: + jempi-pre-processor: + image: jembi/jempi-pre-processor + deploy: + replicas: 1 + resources: + limits: + memory: ${PRE_PROCESSOR_MEMORY_LIMIT} + reservations: + memory: ${PRE_PROCESSOR_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.sync-receiver.yml b/client-registry-jempi/docker-compose.sync-receiver.yml new file mode 100644 index 00000000..230a7888 --- /dev/null +++ b/client-registry-jempi/docker-compose.sync-receiver.yml @@ -0,0 +1,17 @@ +version: '3.9' + +services: + jempi-sync-receiver: + image: jembi/jempi-sync-receiver + ports: + - published: 50040 + target: 50000 + protocol: tcp + mode: host + deploy: + replicas: 1 + resources: + limits: + memory: ${SYNC_RECEIVER_MEMORY_LIMIT} + reservations: + memory: ${SYNC_RECEIVER_MEMORY_RESERVE} diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json index 9e7a1854..23a2b06f 100644 --- a/client-registry-jempi/package-metadata.json +++ b/client-registry-jempi/package-metadata.json @@ -28,6 +28,18 @@ "ALPHA_03_MEMORY_LIMIT": "3G", "ALPHA_03_MEMORY_RESERVE": "500M", "RATEL_MEMORY_LIMIT": "3G", - "RATEL_MEMORY_RESERVE": "500M" + "RATEL_MEMORY_RESERVE": "500M", + "ASYNC_RECEIVER_MEMORY_LIMIT": "3G", + "ASYNC_RECEIVER_MEMORY_RESERVE": "500M", + "EM_CALCULATOR_MEMORY_LIMIT": "3G", + "EM_CALCULATOR_MEMORY_RESERVE": "500M", + "CONTROLLER_MEMORY_LIMIT": "3G", + "CONTROLLER_MEMORY_RESERVE": "500M", + "LINKER_MEMORY_LIMIT": "3G", + "LINKER_MEMORY_RESERVE": "500M", + "PRE_PROCESSOR_MEMORY_LIMIT": "3G", + "PRE_PROCESSOR_MEMORY_RESERVE": "500M", + "SYNC_RECEIVER_MEMORY_LIMIT": "3G", + "SYNC_RECEIVER_MEMORY_RESERVE": "500M" } } From ec4a700777a909853ae3e9c3c638ff9d8bc0eec1 Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Wed, 23 Nov 2022 12:37:59 +0200 Subject: [PATCH 02/11] missed service name prefix for rate PLAT-620 --- client-registry-jempi/docker-compose.dgraph-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-registry-jempi/docker-compose.dgraph-dev.yml b/client-registry-jempi/docker-compose.dgraph-dev.yml index 054675ee..93b6c000 100644 --- a/client-registry-jempi/docker-compose.dgraph-dev.yml +++ b/client-registry-jempi/docker-compose.dgraph-dev.yml @@ -45,7 +45,7 @@ services: protocol: tcp mode: host - ratel: + jempi-ratel: ports: - published: 8010 target: 8000 From 67f3763faa2959410f84abf5315cbf002a2fac7d Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Wed, 23 Nov 2022 12:54:50 +0200 Subject: [PATCH 03/11] combined some services into one compose file these services, for now, can be deployed at the same time, so there is not reason for them to be seperated PLAT-620 --- .../docker-compose.async-receiver.yml | 12 ---- .../docker-compose.combined-dev.yml | 23 +++++++ .../docker-compose.combined.yml | 62 +++++++++++++++++++ .../docker-compose.em-calculator.yml | 12 ---- .../docker-compose.jempi-controller.yml | 17 ----- .../docker-compose.linker.yml | 17 ----- .../docker-compose.pre-processor.yml | 12 ---- .../docker-compose.sync-receiver.yml | 17 ----- 8 files changed, 85 insertions(+), 87 deletions(-) delete mode 100644 client-registry-jempi/docker-compose.async-receiver.yml create mode 100644 client-registry-jempi/docker-compose.combined-dev.yml create mode 100644 client-registry-jempi/docker-compose.combined.yml delete mode 100644 client-registry-jempi/docker-compose.em-calculator.yml delete mode 100644 client-registry-jempi/docker-compose.jempi-controller.yml delete mode 100644 client-registry-jempi/docker-compose.linker.yml delete mode 100644 client-registry-jempi/docker-compose.pre-processor.yml delete mode 100644 client-registry-jempi/docker-compose.sync-receiver.yml diff --git a/client-registry-jempi/docker-compose.async-receiver.yml b/client-registry-jempi/docker-compose.async-receiver.yml deleted file mode 100644 index 393ff0be..00000000 --- a/client-registry-jempi/docker-compose.async-receiver.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: '3.9' - -services: - jempi-async-receiver: - image: jembi/jempi-async-receiver - deploy: - replicas: 1 - resources: - limits: - memory: ${ASYNC_RECEIVER_MEMORY_LIMIT} - reservations: - memory: ${ASYNC_RECEIVER_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.combined-dev.yml b/client-registry-jempi/docker-compose.combined-dev.yml new file mode 100644 index 00000000..a154d235 --- /dev/null +++ b/client-registry-jempi/docker-compose.combined-dev.yml @@ -0,0 +1,23 @@ +version: '3.9' + +services: + jempi-sync-receiver: + ports: + - published: 50040 + target: 50000 + protocol: tcp + mode: host + + jempi-controller: + ports: + - published: 50020 + target: 50000 + protocol: tcp + mode: host + + jempi-linker: + ports: + - published: 50010 + target: 50000 + protocol: tcp + mode: host diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml new file mode 100644 index 00000000..ca0ba7f3 --- /dev/null +++ b/client-registry-jempi/docker-compose.combined.yml @@ -0,0 +1,62 @@ +version: '3.9' + +services: + jempi-async-receiver: + image: jembi/jempi-async-receiver:${ASYNC_RECEIVER_IMAGE_TAG} + deploy: + replicas: 1 + resources: + limits: + memory: ${ASYNC_RECEIVER_MEMORY_LIMIT} + reservations: + memory: ${ASYNC_RECEIVER_MEMORY_RESERVE} + + jempi-sync-receiver: + image: jembi/jempi-sync-receiver:${SYNC_RECEIVER_IMAGE_TAG} + deploy: + replicas: 1 + resources: + limits: + memory: ${SYNC_RECEIVER_MEMORY_LIMIT} + reservations: + memory: ${SYNC_RECEIVER_MEMORY_RESERVE} + + jempi-pre-processor: + image: jembi/jempi-pre-processor:${PRE_PROCESSOR_IMAGE_TAG} + deploy: + replicas: 1 + resources: + limits: + memory: ${PRE_PROCESSOR_MEMORY_LIMIT} + reservations: + memory: ${PRE_PROCESSOR_MEMORY_RESERVE} + + jempi-controller: + image: jembi/jempi-controller:${CONTROLLER_IMAGE_TAG} + deploy: + replicas: 1 + resources: + limits: + memory: ${CONTROLLER_MEMORY_LIMIT} + reservations: + memory: ${CONTROLLER_MEMORY_RESERVE} + + jempi-em-calculator: + image: jembi/jempi-em-calculator:${EM_CALCULATOR_IMAGE_TAG} + deploy: + replicas: 1 + resources: + limits: + memory: ${EM_CALCULATOR_MEMORY_LIMIT} + reservations: + memory: ${EM_CALCULATOR_MEMORY_RESERVE} + + jempi-linker: + image: jembi/jempi-linker:${LINKER_IMAGE_TAG} + deploy: + replicas: 1 + resources: + limits: + memory: ${LINKER_MEMORY_LIMIT} + reservations: + memory: ${LINKER_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.em-calculator.yml b/client-registry-jempi/docker-compose.em-calculator.yml deleted file mode 100644 index bd00e0a1..00000000 --- a/client-registry-jempi/docker-compose.em-calculator.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: '3.9' - -services: - jempi-em-calculator: - image: jembi/jempi-em-calculator - deploy: - replicas: 1 - resources: - limits: - memory: ${EM_CALCULATOR_MEMORY_LIMIT} - reservations: - memory: ${EM_CALCULATOR_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.jempi-controller.yml b/client-registry-jempi/docker-compose.jempi-controller.yml deleted file mode 100644 index f22bb369..00000000 --- a/client-registry-jempi/docker-compose.jempi-controller.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '3.9' - -services: - jempi-controller: - image: jembi/jempi-controller - ports: - - published: 50020 - target: 50000 - protocol: tcp - mode: host - deploy: - replicas: 1 - resources: - limits: - memory: ${CONTROLLER_MEMORY_LIMIT} - reservations: - memory: ${CONTROLLER_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.linker.yml b/client-registry-jempi/docker-compose.linker.yml deleted file mode 100644 index 47bdae81..00000000 --- a/client-registry-jempi/docker-compose.linker.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '3.9' - -services: - jempi-linker: - image: jembi/jempi-linker - ports: - - published: 50010 - target: 50000 - protocol: tcp - mode: host - deploy: - replicas: 1 - resources: - limits: - memory: ${LINKER_MEMORY_LIMIT} - reservations: - memory: ${LINKER_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.pre-processor.yml b/client-registry-jempi/docker-compose.pre-processor.yml deleted file mode 100644 index a924c3aa..00000000 --- a/client-registry-jempi/docker-compose.pre-processor.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: '3.9' - -services: - jempi-pre-processor: - image: jembi/jempi-pre-processor - deploy: - replicas: 1 - resources: - limits: - memory: ${PRE_PROCESSOR_MEMORY_LIMIT} - reservations: - memory: ${PRE_PROCESSOR_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.sync-receiver.yml b/client-registry-jempi/docker-compose.sync-receiver.yml deleted file mode 100644 index 230a7888..00000000 --- a/client-registry-jempi/docker-compose.sync-receiver.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '3.9' - -services: - jempi-sync-receiver: - image: jembi/jempi-sync-receiver - ports: - - published: 50040 - target: 50000 - protocol: tcp - mode: host - deploy: - replicas: 1 - resources: - limits: - memory: ${SYNC_RECEIVER_MEMORY_LIMIT} - reservations: - memory: ${SYNC_RECEIVER_MEMORY_RESERVE} From 54ac2d1d241b79b51991f2859668f008c920a490 Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Wed, 23 Nov 2022 12:59:57 +0200 Subject: [PATCH 04/11] swarm.sh changes to launch some services PLAT-620 --- client-registry-jempi/package-metadata.json | 12 ++++++-- client-registry-jempi/swarm.sh | 34 ++++++++++++++++++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json index 23a2b06f..315eaac8 100644 --- a/client-registry-jempi/package-metadata.json +++ b/client-registry-jempi/package-metadata.json @@ -17,8 +17,6 @@ "JEMPI_KAFDROP_CPU_RESERVE": "0.05", "JEMPI_KAFDROP_MEMORY_LIMIT": "3G", "JEMPI_KAFDROP_MEMORY_RESERVE": "500M", - "JEMPI_KAFKA_TOPICS": "jempi-patient-staging-01,jempi-patient-staging-02,jempi-patient-staging-disi,jempi-patient-controller,jempi-patient-em,jempi-patient-linker,jempi-mu-linker,jempi-journal,jempi-notifications", - "JEMPI_KAFKA_DEBUG": false, "ZERO_01_MEMORY_LIMIT": "3G", "ZERO_01_MEMORY_RESERVE": "500M", "ALPHA_01_MEMORY_LIMIT": "3G", @@ -40,6 +38,14 @@ "PRE_PROCESSOR_MEMORY_LIMIT": "3G", "PRE_PROCESSOR_MEMORY_RESERVE": "500M", "SYNC_RECEIVER_MEMORY_LIMIT": "3G", - "SYNC_RECEIVER_MEMORY_RESERVE": "500M" + "SYNC_RECEIVER_MEMORY_RESERVE": "500M", + "JEMPI_KAFKA_TOPICS": "jempi-patient-staging-01,jempi-patient-staging-02,jempi-patient-staging-disi,jempi-patient-controller,jempi-patient-em,jempi-patient-linker,jempi-mu-linker,jempi-journal,jempi-notifications", + "JEMPI_KAFKA_DEBUG": false, + "ASYNC_RECEIVER_IMAGE_TAG": "0.1.0", + "SYNC_RECEIVER_IMAGE_TAG": "0.1.0", + "PRE_PROCESSOR_IMAGE_TAG": "0.1.0", + "CONTROLLER_IMAGE_TAG": "0.1.0", + "EM_CALCULATOR_IMAGE_TAG": "0.1.0", + "LINKER_IMAGE_TAG": "0.1.0" } } diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh index 703f22e0..8020c5db 100644 --- a/client-registry-jempi/swarm.sh +++ b/client-registry-jempi/swarm.sh @@ -21,10 +21,12 @@ main() { log info "Running Client Registry JeMPI package in DEV mode" kafdrop_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.kafdrop-dev.yml" dgraph_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.dgraph-dev.yml" + combined_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.combined-dev.yml" else log info "Running Client Registry JeMPI package in PROD mode" kafdrop_dev_compose_param="" dgraph_dev_compose_param="" + combined_dev_compose_param="" fi if [[ "$STATEFUL_NODES" == "cluster" ]]; then @@ -67,7 +69,16 @@ main() { docker::await_service_ready jempi-ratel - docker::deploy_sanity "jempi-kafka-01" "jempi-kafka-02" "jempi-kafka-03" "jempi-kafdrop" "jempi-zero-01" "jempi-alpha-01" "jempi-alpha-02" "jempi-alpha-03" "jempi-ratel" + try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.combined.yml $combined_dev_compose_param instant" "Failed to deploy Client Registry - JeMPI" + + docker::await_service_ready jempi-async-receiver + docker::await_service_ready jempi-sync-receiver + docker::await_service_ready jempi-pre-processor + docker::await_service_ready jempi-controller + docker::await_service_ready jempi-em-calculator + docker::await_service_ready jempi-linker + + docker::deploy_sanity "jempi-kafka-01" "jempi-kafka-02" "jempi-kafka-03" "jempi-kafdrop" "jempi-zero-01" "jempi-alpha-01" "jempi-alpha-02" "jempi-alpha-03" "jempi-ratel" "jempi-async-receiver" "jempi-sync-receiver" "jempi-pre-processor" "jempi-controller" "jempi-em-calculator" "jempi-linker" elif [[ "${ACTION}" == "down" ]]; then log info "Scaling down client-registry-jempi" @@ -84,6 +95,13 @@ main() { try "docker service scale instant_jempi-alpha-03=0" "Failed to scale down jempi-alpha-03" try "docker service scale instant_jempi-ratel=0" "Failed to scale down jempi-ratel" + + try "docker service scale instant_jempi-async-receiver=0" "Failed to scale down jempi-async-receiver" + try "docker service scale instant_jempi-sync-receiver=0" "Failed to scale down jempi-sync-receiver" + try "docker service scale instant_jempi-pre-processor=0" "Failed to scale down jempi-pre-processor" + try "docker service scale instant_jempi-controller=0" "Failed to scale down jempi-controller" + try "docker service scale instant_jempi-em-calculator=0" "Failed to scale down jempi-em-calculator" + try "docker service scale instant_jempi-linker=0" "Failed to scale down jempi-linker" elif [[ "${ACTION}" == "destroy" ]]; then log warn "Volumes are only deleted on the host on which the command is run. Volumes on other nodes are not deleted" @@ -103,6 +121,13 @@ main() { docker::service_destroy jempi-ratel + docker::service_destroy jempi-async-receiver + docker::service_destroy jempi-sync-receiver + docker::service_destroy jempi-pre-processor + docker::service_destroy jempi-controller + docker::service_destroy jempi-em-calculator + docker::service_destroy jempi-linker + docker::try_remove_volume jempi-kafka-01-data docker::try_remove_volume jempi-kafka-02-data docker::try_remove_volume jempi-kafka-03-data @@ -120,3 +145,10 @@ main() { } main "$@" + +jempi-async-receiver +jempi-sync-receiver +jempi-pre-processor +jempi-controller +jempi-em-calculator +jempi-linker From 1eaaf897f1e9c621711925c15e16e75e0cd29c94 Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Wed, 23 Nov 2022 13:00:58 +0200 Subject: [PATCH 05/11] aligned kafka topics with JeMPI repo PLAT-620 --- client-registry-jempi/package-metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json index 315eaac8..a19d48ee 100644 --- a/client-registry-jempi/package-metadata.json +++ b/client-registry-jempi/package-metadata.json @@ -39,7 +39,7 @@ "PRE_PROCESSOR_MEMORY_RESERVE": "500M", "SYNC_RECEIVER_MEMORY_LIMIT": "3G", "SYNC_RECEIVER_MEMORY_RESERVE": "500M", - "JEMPI_KAFKA_TOPICS": "jempi-patient-staging-01,jempi-patient-staging-02,jempi-patient-staging-disi,jempi-patient-controller,jempi-patient-em,jempi-patient-linker,jempi-mu-linker,jempi-journal,jempi-notifications", + "JEMPI_KAFKA_TOPICS": "JeMPI-async-preprocessor,JeMPI-patient-controller,JeMPI-patient-em,JeMPI-patient-linker,JeMPI-mu-linker,JeMPI-notifications", "JEMPI_KAFKA_DEBUG": false, "ASYNC_RECEIVER_IMAGE_TAG": "0.1.0", "SYNC_RECEIVER_IMAGE_TAG": "0.1.0", From c219609aaf31c45eec25c65306ae25cc5c225232 Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Wed, 23 Nov 2022 13:15:43 +0200 Subject: [PATCH 06/11] removed accidentally inserted code PLAT-620 --- client-registry-jempi/swarm.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh index 8020c5db..e5501680 100644 --- a/client-registry-jempi/swarm.sh +++ b/client-registry-jempi/swarm.sh @@ -145,10 +145,3 @@ main() { } main "$@" - -jempi-async-receiver -jempi-sync-receiver -jempi-pre-processor -jempi-controller -jempi-em-calculator -jempi-linker From bb25206bd1fda1836dd69ad3bda30d07e0750a2f Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Thu, 24 Nov 2022 10:23:03 +0200 Subject: [PATCH 07/11] added jempi-api to MPI package PLAT-620 --- .../docker-compose.api-dev.yml | 9 +++++++++ client-registry-jempi/docker-compose.api.yml | 12 +++++++++++ client-registry-jempi/package-metadata.json | 5 ++++- client-registry-jempi/swarm.sh | 20 ++++++++++++++----- 4 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 client-registry-jempi/docker-compose.api-dev.yml create mode 100644 client-registry-jempi/docker-compose.api.yml diff --git a/client-registry-jempi/docker-compose.api-dev.yml b/client-registry-jempi/docker-compose.api-dev.yml new file mode 100644 index 00000000..9fdd32ca --- /dev/null +++ b/client-registry-jempi/docker-compose.api-dev.yml @@ -0,0 +1,9 @@ +version: '3.9' + +services: + jempi-api: + ports: + - published: 50000 + target: 50000 + protocol: tcp + mode: host diff --git a/client-registry-jempi/docker-compose.api.yml b/client-registry-jempi/docker-compose.api.yml new file mode 100644 index 00000000..bd2c1218 --- /dev/null +++ b/client-registry-jempi/docker-compose.api.yml @@ -0,0 +1,12 @@ +version: '3.9' + +services: + jempi-api: + image: jembi/jempi-api:${API_IMAGE_TAG} + deploy: + replicas: 1 + resources: + limits: + memory: ${API_MEMORY_LIMIT} + reservations: + memory: ${API_MEMORY_RESERVE} diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json index a19d48ee..a00bdb14 100644 --- a/client-registry-jempi/package-metadata.json +++ b/client-registry-jempi/package-metadata.json @@ -39,6 +39,8 @@ "PRE_PROCESSOR_MEMORY_RESERVE": "500M", "SYNC_RECEIVER_MEMORY_LIMIT": "3G", "SYNC_RECEIVER_MEMORY_RESERVE": "500M", + "API_MEMORY_LIMIT": "3G", + "API_MEMORY_RESERVE": "500M", "JEMPI_KAFKA_TOPICS": "JeMPI-async-preprocessor,JeMPI-patient-controller,JeMPI-patient-em,JeMPI-patient-linker,JeMPI-mu-linker,JeMPI-notifications", "JEMPI_KAFKA_DEBUG": false, "ASYNC_RECEIVER_IMAGE_TAG": "0.1.0", @@ -46,6 +48,7 @@ "PRE_PROCESSOR_IMAGE_TAG": "0.1.0", "CONTROLLER_IMAGE_TAG": "0.1.0", "EM_CALCULATOR_IMAGE_TAG": "0.1.0", - "LINKER_IMAGE_TAG": "0.1.0" + "LINKER_IMAGE_TAG": "0.1.0", + "API_IMAGE_TAG": "0.1.0" } } diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh index e5501680..c0a0290b 100644 --- a/client-registry-jempi/swarm.sh +++ b/client-registry-jempi/swarm.sh @@ -22,11 +22,13 @@ main() { kafdrop_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.kafdrop-dev.yml" dgraph_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.dgraph-dev.yml" combined_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.combined-dev.yml" + api_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.api-dev.yml" else log info "Running Client Registry JeMPI package in PROD mode" kafdrop_dev_compose_param="" dgraph_dev_compose_param="" combined_dev_compose_param="" + api_dev_compose_param="" fi if [[ "$STATEFUL_NODES" == "cluster" ]]; then @@ -38,13 +40,13 @@ main() { fi if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then - try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.kafka.yml instant" "Failed to deploy Client Registry - JeMPI" + try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.kafka.yml instant" "Failed to deploy Client Registry - JeMPI (kafka.yml)" docker::await_service_ready jempi-kafka-01 docker::await_service_ready jempi-kafka-02 docker::await_service_ready jempi-kafka-03 - try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.kafdrop.yml $kafdrop_dev_compose_param instant" "Failed to deploy jempi-Kafdrop" + try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.kafdrop.yml $kafdrop_dev_compose_param instant" "Failed to deploy Client Registry - JeMPI (kafdrop.yml)" docker::await_service_ready jempi-kafdrop @@ -59,7 +61,7 @@ main() { config::remove_stale_service_configs "${COMPOSE_FILE_PATH}"/importer/docker-compose.config.yml "jempi-kafka" - try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.dgraph.yml $dgraph_dev_compose_param $dgraph_cluster_compose_param instant" "Failed to deploy Client Registry - JeMPI" + try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.dgraph.yml $dgraph_dev_compose_param $dgraph_cluster_compose_param instant" "Failed to deploy Client Registry - JeMPI (dgraph.yml)" docker::await_service_ready jempi-zero-01 @@ -69,7 +71,7 @@ main() { docker::await_service_ready jempi-ratel - try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.combined.yml $combined_dev_compose_param instant" "Failed to deploy Client Registry - JeMPI" + try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.combined.yml $combined_dev_compose_param instant" "Failed to deploy Client Registry - JeMPI (combined.yml)" docker::await_service_ready jempi-async-receiver docker::await_service_ready jempi-sync-receiver @@ -78,7 +80,11 @@ main() { docker::await_service_ready jempi-em-calculator docker::await_service_ready jempi-linker - docker::deploy_sanity "jempi-kafka-01" "jempi-kafka-02" "jempi-kafka-03" "jempi-kafdrop" "jempi-zero-01" "jempi-alpha-01" "jempi-alpha-02" "jempi-alpha-03" "jempi-ratel" "jempi-async-receiver" "jempi-sync-receiver" "jempi-pre-processor" "jempi-controller" "jempi-em-calculator" "jempi-linker" + try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.api.yml $api_dev_compose_param instant" "Failed to deploy Client Registry - JeMPI (api.yml)" + + docker::await_service_ready jempi-api + + docker::deploy_sanity "jempi-kafka-01" "jempi-kafka-02" "jempi-kafka-03" "jempi-kafdrop" "jempi-zero-01" "jempi-alpha-01" "jempi-alpha-02" "jempi-alpha-03" "jempi-ratel" "jempi-async-receiver" "jempi-sync-receiver" "jempi-pre-processor" "jempi-controller" "jempi-em-calculator" "jempi-linker" "jempi-api" elif [[ "${ACTION}" == "down" ]]; then log info "Scaling down client-registry-jempi" @@ -102,6 +108,8 @@ main() { try "docker service scale instant_jempi-controller=0" "Failed to scale down jempi-controller" try "docker service scale instant_jempi-em-calculator=0" "Failed to scale down jempi-em-calculator" try "docker service scale instant_jempi-linker=0" "Failed to scale down jempi-linker" + + try "docker service scale instant_jempi-api=0" "Failed to scale down jempi-api" elif [[ "${ACTION}" == "destroy" ]]; then log warn "Volumes are only deleted on the host on which the command is run. Volumes on other nodes are not deleted" @@ -128,6 +136,8 @@ main() { docker::service_destroy jempi-em-calculator docker::service_destroy jempi-linker + docker::service_destroy jempi-api + docker::try_remove_volume jempi-kafka-01-data docker::try_remove_volume jempi-kafka-02-data docker::try_remove_volume jempi-kafka-03-data From 00521f9488b1460e542f5446f5a18559ec833dc0 Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Thu, 24 Nov 2022 13:00:36 +0200 Subject: [PATCH 08/11] loop through service names for jempi some commands were being done repetitively, so they are no done in a loop PLAT-620 --- client-registry-jempi/swarm.sh | 76 +++++++++++++--------------------- 1 file changed, 28 insertions(+), 48 deletions(-) diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh index c0a0290b..4cb8639d 100644 --- a/client-registry-jempi/swarm.sh +++ b/client-registry-jempi/swarm.sh @@ -16,6 +16,26 @@ readonly ROOT_PATH . "${ROOT_PATH}/utils/docker-utils.sh" . "${ROOT_PATH}/utils/log.sh" +service_names=( + "jempi-kafka-01" + "jempi-kafka-02" + "jempi-kafka-03" + "jempi-kafdrop" + "jempi-zero-01" + "jempi-alpha-01" + "jempi-alpha-02" + "jempi-alpha-03" + "jempi-ratel" + "jempi-async-receiver" + "jempi-sync-receiver" + "jempi-pre-processor" + "jempi-controller" + "jempi-em-calculator" + "jempi-linker" + "jempi-api" +) +readonly service_names + main() { if [[ "$MODE" == "dev" ]]; then log info "Running Client Registry JeMPI package in DEV mode" @@ -83,60 +103,20 @@ main() { try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.api.yml $api_dev_compose_param instant" "Failed to deploy Client Registry - JeMPI (api.yml)" docker::await_service_ready jempi-api - - docker::deploy_sanity "jempi-kafka-01" "jempi-kafka-02" "jempi-kafka-03" "jempi-kafdrop" "jempi-zero-01" "jempi-alpha-01" "jempi-alpha-02" "jempi-alpha-03" "jempi-ratel" "jempi-async-receiver" "jempi-sync-receiver" "jempi-pre-processor" "jempi-controller" "jempi-em-calculator" "jempi-linker" "jempi-api" + + docker::deploy_sanity "${service_names[@]}" elif [[ "${ACTION}" == "down" ]]; then log info "Scaling down client-registry-jempi" - try "docker service scale instant_jempi-kafka-01=0" "Failed to scale down jempi-kafka-01" - try "docker service scale instant_jempi-kafka-02=0" "Failed to scale down jempi-kafka-02" - try "docker service scale instant_jempi-kafka-03=0" "Failed to scale down jempi-kafka-03" - - try "docker service scale instant_jempi-kafdrop=0" "Failed to scale down jempi-kafdrop" - - try "docker service scale instant_jempi-zero-01=0" "Failed to scale down jempi-zero-01" - - try "docker service scale instant_jempi-alpha-01=0" "Failed to scale down jempi-alpha-01" - try "docker service scale instant_jempi-alpha-02=0" "Failed to scale down jempi-alpha-02" - try "docker service scale instant_jempi-alpha-03=0" "Failed to scale down jempi-alpha-03" - - try "docker service scale instant_jempi-ratel=0" "Failed to scale down jempi-ratel" - - try "docker service scale instant_jempi-async-receiver=0" "Failed to scale down jempi-async-receiver" - try "docker service scale instant_jempi-sync-receiver=0" "Failed to scale down jempi-sync-receiver" - try "docker service scale instant_jempi-pre-processor=0" "Failed to scale down jempi-pre-processor" - try "docker service scale instant_jempi-controller=0" "Failed to scale down jempi-controller" - try "docker service scale instant_jempi-em-calculator=0" "Failed to scale down jempi-em-calculator" - try "docker service scale instant_jempi-linker=0" "Failed to scale down jempi-linker" - - try "docker service scale instant_jempi-api=0" "Failed to scale down jempi-api" + for service_name in "${service_names[@]}"; do + try "docker service scale instant_$service_name=0" "Failed to scale down $service_name" + done elif [[ "${ACTION}" == "destroy" ]]; then log warn "Volumes are only deleted on the host on which the command is run. Volumes on other nodes are not deleted" - docker::service_destroy jempi-kafka-01 - docker::service_destroy jempi-kafka-02 - docker::service_destroy jempi-kafka-03 - - docker::service_destroy jempi-kafdrop - - docker::service_destroy jempi-kafka-config-importer - - docker::service_destroy jempi-zero-01 - - docker::service_destroy jempi-alpha-01 - docker::service_destroy jempi-alpha-02 - docker::service_destroy jempi-alpha-03 - - docker::service_destroy jempi-ratel - - docker::service_destroy jempi-async-receiver - docker::service_destroy jempi-sync-receiver - docker::service_destroy jempi-pre-processor - docker::service_destroy jempi-controller - docker::service_destroy jempi-em-calculator - docker::service_destroy jempi-linker - - docker::service_destroy jempi-api + for service_name in "${service_names[@]}"; do + docker::service_destroy "$service_name" + done docker::try_remove_volume jempi-kafka-01-data docker::try_remove_volume jempi-kafka-02-data From 182f3a0eae9895b8cd01d4377ce08ff4dcea56be Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Thu, 24 Nov 2022 13:01:28 +0200 Subject: [PATCH 09/11] JeMPI launches dgraph-zero first there are possible errors that can arise form launching the alphas before the zero is up, so the zero is launched in a seperated compose file PLAT-620 --- .../docker-compose.dgraph-cluster.yml | 6 ------ .../docker-compose.dgraph-dev.yml | 11 ---------- .../docker-compose.dgraph-zero-cluster.yml | 8 +++++++ .../docker-compose.dgraph-zero-dev.yml | 13 ++++++++++++ .../docker-compose.dgraph-zero.yml | 21 +++++++++++++++++++ .../docker-compose.dgraph.yml | 16 -------------- client-registry-jempi/swarm.sh | 8 ++++++- 7 files changed, 49 insertions(+), 34 deletions(-) create mode 100644 client-registry-jempi/docker-compose.dgraph-zero-cluster.yml create mode 100644 client-registry-jempi/docker-compose.dgraph-zero-dev.yml create mode 100644 client-registry-jempi/docker-compose.dgraph-zero.yml diff --git a/client-registry-jempi/docker-compose.dgraph-cluster.yml b/client-registry-jempi/docker-compose.dgraph-cluster.yml index 32c9da6b..86898a77 100644 --- a/client-registry-jempi/docker-compose.dgraph-cluster.yml +++ b/client-registry-jempi/docker-compose.dgraph-cluster.yml @@ -1,12 +1,6 @@ version: '3.9' services: - jempi-zero-01: - deploy: - placement: - constraints: - - node.hostname == node-1 - jempi-alpha-01: deploy: placement: diff --git a/client-registry-jempi/docker-compose.dgraph-dev.yml b/client-registry-jempi/docker-compose.dgraph-dev.yml index 93b6c000..51be6bac 100644 --- a/client-registry-jempi/docker-compose.dgraph-dev.yml +++ b/client-registry-jempi/docker-compose.dgraph-dev.yml @@ -1,17 +1,6 @@ version: '3.9' services: - jempi-zero-01: - ports: - - published: 5080 - target: 5080 - protocol: tcp - mode: host - - published: 6080 - target: 6080 - protocol: tcp - mode: host - jempi-alpha-01: ports: - published: 8070 diff --git a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml new file mode 100644 index 00000000..ca029604 --- /dev/null +++ b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml @@ -0,0 +1,8 @@ +version: '3.9' + +services: + jempi-zero-01: + deploy: + placement: + constraints: + - node.hostname == node-1 diff --git a/client-registry-jempi/docker-compose.dgraph-zero-dev.yml b/client-registry-jempi/docker-compose.dgraph-zero-dev.yml new file mode 100644 index 00000000..43f49501 --- /dev/null +++ b/client-registry-jempi/docker-compose.dgraph-zero-dev.yml @@ -0,0 +1,13 @@ +version: '3.9' + +services: + jempi-zero-01: + ports: + - published: 5080 + target: 5080 + protocol: tcp + mode: host + - published: 6080 + target: 6080 + protocol: tcp + mode: host diff --git a/client-registry-jempi/docker-compose.dgraph-zero.yml b/client-registry-jempi/docker-compose.dgraph-zero.yml new file mode 100644 index 00000000..6eb7d937 --- /dev/null +++ b/client-registry-jempi/docker-compose.dgraph-zero.yml @@ -0,0 +1,21 @@ +version: '3.9' + +services: + jempi-zero-01: + image: dgraph/dgraph:v22.0.0 + volumes: + - jempi-zero-01-data:/dgraph + deploy: + replicas: 1 + resources: + limits: + memory: ${ZERO_01_MEMORY_LIMIT} + reservations: + memory: ${ZERO_01_MEMORY_RESERVE} + restart_policy: + condition: on-failure + command: dgraph zero --my=jempi-zero-01:5080 --replicas 1 + +volumes: + jempi-zero-01-data: + diff --git a/client-registry-jempi/docker-compose.dgraph.yml b/client-registry-jempi/docker-compose.dgraph.yml index 048c6563..d88c4ec8 100644 --- a/client-registry-jempi/docker-compose.dgraph.yml +++ b/client-registry-jempi/docker-compose.dgraph.yml @@ -1,21 +1,6 @@ version: '3.9' services: - jempi-zero-01: - image: dgraph/dgraph:v22.0.0 - volumes: - - jempi-zero-01-data:/dgraph - deploy: - replicas: 1 - resources: - limits: - memory: ${ZERO_01_MEMORY_LIMIT} - reservations: - memory: ${ZERO_01_MEMORY_RESERVE} - restart_policy: - condition: on-failure - command: dgraph zero --my=jempi-zero-01:5080 --replicas 1 - jempi-alpha-01: image: dgraph/dgraph:v22.0.0 volumes: @@ -75,7 +60,6 @@ services: command: dgraph-ratel volumes: - jempi-zero-01-data: jempi-alpha-01-data: jempi-alpha-02-data: jempi-alpha-03-data: diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh index 4cb8639d..6b301b1c 100644 --- a/client-registry-jempi/swarm.sh +++ b/client-registry-jempi/swarm.sh @@ -41,12 +41,14 @@ main() { log info "Running Client Registry JeMPI package in DEV mode" kafdrop_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.kafdrop-dev.yml" dgraph_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.dgraph-dev.yml" + dgraph_zero_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.dgraph-zero-dev.yml" combined_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.combined-dev.yml" api_dev_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.api-dev.yml" else log info "Running Client Registry JeMPI package in PROD mode" kafdrop_dev_compose_param="" dgraph_dev_compose_param="" + dgraph_zero_dev_compose_param="" combined_dev_compose_param="" api_dev_compose_param="" fi @@ -54,9 +56,11 @@ main() { if [[ "$STATEFUL_NODES" == "cluster" ]]; then log info "Running in clustered mode" dgraph_cluster_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.dgraph-cluster.yml" + dgraph_zero_cluster_compose_param="-c ${COMPOSE_FILE_PATH}/docker-compose.dgraph-zero-cluster.yml" else log info "Running in single-node mode" dgraph_cluster_compose_param="" + dgraph_zero_cluster_compose_param="" fi if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then @@ -81,10 +85,12 @@ main() { config::remove_stale_service_configs "${COMPOSE_FILE_PATH}"/importer/docker-compose.config.yml "jempi-kafka" - try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.dgraph.yml $dgraph_dev_compose_param $dgraph_cluster_compose_param instant" "Failed to deploy Client Registry - JeMPI (dgraph.yml)" + try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.dgraph-zero.yml $dgraph_zero_dev_compose_param $dgraph_zero_cluster_compose_param instant" "Failed to deploy Client Registry - JeMPI (dgraph-zero.yml)" docker::await_service_ready jempi-zero-01 + try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.dgraph.yml $dgraph_dev_compose_param $dgraph_cluster_compose_param instant" "Failed to deploy Client Registry - JeMPI (dgraph.yml)" + docker::await_service_ready jempi-alpha-01 docker::await_service_ready jempi-alpha-02 docker::await_service_ready jempi-alpha-03 From 8cf0da728ebf6c3e0334439c485d0bbdfdc77f73 Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Thu, 24 Nov 2022 13:03:29 +0200 Subject: [PATCH 10/11] jeMPI package in config.yml PLAT-620 --- config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.yml b/config.yml index d4341dd3..5f5504ca 100644 --- a/config.yml +++ b/config.yml @@ -40,3 +40,5 @@ packages: name: Monitoring package - id: mpi-mediator name: OpenHIM-Sante Mediator + - id: client-registry-jempi + name: Client Registry - JeMPI From 93f48469da08e799a0f0e45406afb00128ecb8ad Mon Sep 17 00:00:00 2001 From: mark-labuschagne Date: Thu, 24 Nov 2022 14:00:40 +0200 Subject: [PATCH 11/11] prefixed env vars with JEMPI_ PLAT-620 --- client-registry-jempi/docker-compose.api.yml | 6 +- .../docker-compose.combined.yml | 36 ++++----- .../docker-compose.dgraph-zero.yml | 5 +- .../docker-compose.dgraph.yml | 16 ++-- .../docker-compose.kafka.yml | 12 +-- client-registry-jempi/package-metadata.json | 74 +++++++++---------- 6 files changed, 75 insertions(+), 74 deletions(-) diff --git a/client-registry-jempi/docker-compose.api.yml b/client-registry-jempi/docker-compose.api.yml index bd2c1218..9e33e217 100644 --- a/client-registry-jempi/docker-compose.api.yml +++ b/client-registry-jempi/docker-compose.api.yml @@ -2,11 +2,11 @@ version: '3.9' services: jempi-api: - image: jembi/jempi-api:${API_IMAGE_TAG} + image: jembi/jempi-api:${JEMPI_API_IMAGE_TAG} deploy: replicas: 1 resources: limits: - memory: ${API_MEMORY_LIMIT} + memory: ${JEMPI_API_MEMORY_LIMIT} reservations: - memory: ${API_MEMORY_RESERVE} + memory: ${JEMPI_API_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml index ca0ba7f3..445965c1 100644 --- a/client-registry-jempi/docker-compose.combined.yml +++ b/client-registry-jempi/docker-compose.combined.yml @@ -2,61 +2,61 @@ version: '3.9' services: jempi-async-receiver: - image: jembi/jempi-async-receiver:${ASYNC_RECEIVER_IMAGE_TAG} + image: jembi/jempi-async-receiver:${JEMPI_AJEMPI_SYNC_RECEIVER_IMAGE_TAG} deploy: replicas: 1 resources: limits: - memory: ${ASYNC_RECEIVER_MEMORY_LIMIT} + memory: ${JEMPI_ASYNC_RECEIVER_MEMORY_LIMIT} reservations: - memory: ${ASYNC_RECEIVER_MEMORY_RESERVE} + memory: ${JEMPI_ASYNC_RECEIVER_MEMORY_RESERVE} jempi-sync-receiver: - image: jembi/jempi-sync-receiver:${SYNC_RECEIVER_IMAGE_TAG} + image: jembi/jempi-sync-receiver:${JEMPI_SYNC_RECEIVER_IMAGE_TAG} deploy: replicas: 1 resources: limits: - memory: ${SYNC_RECEIVER_MEMORY_LIMIT} + memory: ${JEMPI_SYNC_RECEIVER_MEMORY_LIMIT} reservations: - memory: ${SYNC_RECEIVER_MEMORY_RESERVE} + memory: ${JEMPI_SYNC_RECEIVER_MEMORY_RESERVE} jempi-pre-processor: - image: jembi/jempi-pre-processor:${PRE_PROCESSOR_IMAGE_TAG} + image: jembi/jempi-pre-processor:${JEMPI_PRE_PROCESSOR_IMAGE_TAG} deploy: replicas: 1 resources: limits: - memory: ${PRE_PROCESSOR_MEMORY_LIMIT} + memory: ${JEMPI_PRE_PROCESSOR_MEMORY_LIMIT} reservations: - memory: ${PRE_PROCESSOR_MEMORY_RESERVE} + memory: ${JEMPI_PRE_PROCESSOR_MEMORY_RESERVE} jempi-controller: - image: jembi/jempi-controller:${CONTROLLER_IMAGE_TAG} + image: jembi/jempi-controller:${JEMPI_CONTROLLER_IMAGE_TAG} deploy: replicas: 1 resources: limits: - memory: ${CONTROLLER_MEMORY_LIMIT} + memory: ${JEMPI_CONTROLLER_MEMORY_LIMIT} reservations: - memory: ${CONTROLLER_MEMORY_RESERVE} + memory: ${JEMPI_CONTROLLER_MEMORY_RESERVE} jempi-em-calculator: - image: jembi/jempi-em-calculator:${EM_CALCULATOR_IMAGE_TAG} + image: jembi/jempi-em-calculator:${JEMPI_EM_CALCULATOR_IMAGE_TAG} deploy: replicas: 1 resources: limits: - memory: ${EM_CALCULATOR_MEMORY_LIMIT} + memory: ${JEMPI_EM_CALCULATOR_MEMORY_LIMIT} reservations: - memory: ${EM_CALCULATOR_MEMORY_RESERVE} + memory: ${JEMPI_EM_CALCULATOR_MEMORY_RESERVE} jempi-linker: - image: jembi/jempi-linker:${LINKER_IMAGE_TAG} + image: jembi/jempi-linker:${JEMPI_LINKER_IMAGE_TAG} deploy: replicas: 1 resources: limits: - memory: ${LINKER_MEMORY_LIMIT} + memory: ${JEMPI_LINKER_MEMORY_LIMIT} reservations: - memory: ${LINKER_MEMORY_RESERVE} + memory: ${JEMPI_LINKER_MEMORY_RESERVE} diff --git a/client-registry-jempi/docker-compose.dgraph-zero.yml b/client-registry-jempi/docker-compose.dgraph-zero.yml index 6eb7d937..4739a3ba 100644 --- a/client-registry-jempi/docker-compose.dgraph-zero.yml +++ b/client-registry-jempi/docker-compose.dgraph-zero.yml @@ -9,9 +9,9 @@ services: replicas: 1 resources: limits: - memory: ${ZERO_01_MEMORY_LIMIT} + memory: ${JEMPI_ZERO_01_MEMORY_LIMIT} reservations: - memory: ${ZERO_01_MEMORY_RESERVE} + memory: ${JEMPI_ZERO_01_MEMORY_RESERVE} restart_policy: condition: on-failure command: dgraph zero --my=jempi-zero-01:5080 --replicas 1 @@ -19,3 +19,4 @@ services: volumes: jempi-zero-01-data: + diff --git a/client-registry-jempi/docker-compose.dgraph.yml b/client-registry-jempi/docker-compose.dgraph.yml index d88c4ec8..6ca9d097 100644 --- a/client-registry-jempi/docker-compose.dgraph.yml +++ b/client-registry-jempi/docker-compose.dgraph.yml @@ -9,9 +9,9 @@ services: replicas: 1 resources: limits: - memory: ${ALPHA_01_MEMORY_LIMIT} + memory: ${JEMPI_ALPHA_01_MEMORY_LIMIT} reservations: - memory: ${ALPHA_01_MEMORY_RESERVE} + memory: ${JEMPI_ALPHA_01_MEMORY_RESERVE} restart_policy: condition: on-failure command: dgraph alpha --my=jempi-alpha-01:7080 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 --telemetry "sentry=false;" @@ -24,9 +24,9 @@ services: replicas: 1 resources: limits: - memory: ${ALPHA_02_MEMORY_LIMIT} + memory: ${JEMPI_ALPHA_02_MEMORY_LIMIT} reservations: - memory: ${ALPHA_02_MEMORY_RESERVE} + memory: ${JEMPI_ALPHA_02_MEMORY_RESERVE} restart_policy: condition: on-failure command: dgraph alpha --my=jempi-alpha-02:7081 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 1 --telemetry "sentry=false;" @@ -39,9 +39,9 @@ services: replicas: 1 resources: limits: - memory: ${ALPHA_03_MEMORY_LIMIT} + memory: ${JEMPI_ALPHA_03_MEMORY_LIMIT} reservations: - memory: ${ALPHA_03_MEMORY_RESERVE} + memory: ${JEMPI_ALPHA_03_MEMORY_RESERVE} restart_policy: condition: on-failure command: dgraph alpha --my=jempi-alpha-03:7082 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 2 --telemetry "sentry=false;" @@ -52,9 +52,9 @@ services: replicas: 1 resources: limits: - memory: ${RATEL_MEMORY_LIMIT} + memory: ${JEMPI_RATEL_MEMORY_LIMIT} reservations: - memory: ${RATEL_MEMORY_RESERVE} + memory: ${JEMPI_RATEL_MEMORY_RESERVE} restart_policy: condition: on-failure command: dgraph-ratel diff --git a/client-registry-jempi/docker-compose.kafka.yml b/client-registry-jempi/docker-compose.kafka.yml index 787d71d4..f1ca8726 100644 --- a/client-registry-jempi/docker-compose.kafka.yml +++ b/client-registry-jempi/docker-compose.kafka.yml @@ -23,9 +23,9 @@ services: replicas: 1 resources: limits: - memory: ${KAFKA_01_MEMORY_LIMIT} + memory: ${JEMPI_KAFKA_01_MEMORY_LIMIT} reservations: - memory: ${KAFKA_01_MEMORY_RESERVE} + memory: ${JEMPI_KAFKA_01_MEMORY_RESERVE} restart_policy: condition: on-failure @@ -51,9 +51,9 @@ services: replicas: 1 resources: limits: - memory: ${KAFKA_02_MEMORY_LIMIT} + memory: ${JEMPI_KAFKA_02_MEMORY_LIMIT} reservations: - memory: ${KAFKA_02_MEMORY_RESERVE} + memory: ${JEMPI_KAFKA_02_MEMORY_RESERVE} restart_policy: condition: on-failure @@ -79,9 +79,9 @@ services: replicas: 1 resources: limits: - memory: ${KAFKA_03_MEMORY_LIMIT} + memory: ${JEMPI_KAFKA_03_MEMORY_LIMIT} reservations: - memory: ${KAFKA_03_MEMORY_RESERVE} + memory: ${JEMPI_KAFKA_03_MEMORY_RESERVE} restart_policy: condition: on-failure diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json index a00bdb14..67cb5994 100644 --- a/client-registry-jempi/package-metadata.json +++ b/client-registry-jempi/package-metadata.json @@ -7,48 +7,48 @@ "dependencies": [], "environmentVariables": { "STATEFUL_NODES": "single", - "KAFKA_01_MEMORY_LIMIT": "3G", - "KAFKA_01_MEMORY_RESERVE": "500M", - "KAFKA_02_MEMORY_LIMIT": "3G", - "KAFKA_02_MEMORY_RESERVE": "500M", - "KAFKA_03_MEMORY_LIMIT": "3G", - "KAFKA_03_MEMORY_RESERVE": "500M", + "JEMPI_KAFKA_01_MEMORY_LIMIT": "3G", + "JEMPI_KAFKA_01_MEMORY_RESERVE": "500M", + "JEMPI_KAFKA_02_MEMORY_LIMIT": "3G", + "JEMPI_KAFKA_02_MEMORY_RESERVE": "500M", + "JEMPI_KAFKA_03_MEMORY_LIMIT": "3G", + "JEMPI_KAFKA_03_MEMORY_RESERVE": "500M", "JEMPI_KAFDROP_CPU_LIMIT": "0", "JEMPI_KAFDROP_CPU_RESERVE": "0.05", "JEMPI_KAFDROP_MEMORY_LIMIT": "3G", "JEMPI_KAFDROP_MEMORY_RESERVE": "500M", - "ZERO_01_MEMORY_LIMIT": "3G", - "ZERO_01_MEMORY_RESERVE": "500M", - "ALPHA_01_MEMORY_LIMIT": "3G", - "ALPHA_01_MEMORY_RESERVE": "500M", - "ALPHA_02_MEMORY_LIMIT": "3G", - "ALPHA_02_MEMORY_RESERVE": "500M", - "ALPHA_03_MEMORY_LIMIT": "3G", - "ALPHA_03_MEMORY_RESERVE": "500M", - "RATEL_MEMORY_LIMIT": "3G", - "RATEL_MEMORY_RESERVE": "500M", - "ASYNC_RECEIVER_MEMORY_LIMIT": "3G", - "ASYNC_RECEIVER_MEMORY_RESERVE": "500M", - "EM_CALCULATOR_MEMORY_LIMIT": "3G", - "EM_CALCULATOR_MEMORY_RESERVE": "500M", - "CONTROLLER_MEMORY_LIMIT": "3G", - "CONTROLLER_MEMORY_RESERVE": "500M", - "LINKER_MEMORY_LIMIT": "3G", - "LINKER_MEMORY_RESERVE": "500M", - "PRE_PROCESSOR_MEMORY_LIMIT": "3G", - "PRE_PROCESSOR_MEMORY_RESERVE": "500M", - "SYNC_RECEIVER_MEMORY_LIMIT": "3G", - "SYNC_RECEIVER_MEMORY_RESERVE": "500M", - "API_MEMORY_LIMIT": "3G", - "API_MEMORY_RESERVE": "500M", + "JEMPI_ZERO_01_MEMORY_LIMIT": "3G", + "JEMPI_ZERO_01_MEMORY_RESERVE": "500M", + "JEMPI_ALPHA_01_MEMORY_LIMIT": "3G", + "JEMPI_ALPHA_01_MEMORY_RESERVE": "500M", + "JEMPI_ALPHA_02_MEMORY_LIMIT": "3G", + "JEMPI_ALPHA_02_MEMORY_RESERVE": "500M", + "JEMPI_ALPHA_03_MEMORY_LIMIT": "3G", + "JEMPI_ALPHA_03_MEMORY_RESERVE": "500M", + "JEMPI_RATEL_MEMORY_LIMIT": "3G", + "JEMPI_RATEL_MEMORY_RESERVE": "500M", + "JEMPI_ASYNC_RECEIVER_MEMORY_LIMIT": "3G", + "JEMPI_ASYNC_RECEIVER_MEMORY_RESERVE": "500M", + "JEMPI_EM_CALCULATOR_MEMORY_LIMIT": "3G", + "JEMPI_EM_CALCULATOR_MEMORY_RESERVE": "500M", + "JEMPI_CONTROLLER_MEMORY_LIMIT": "3G", + "JEMPI_CONTROLLER_MEMORY_RESERVE": "500M", + "JEMPI_LINKER_MEMORY_LIMIT": "3G", + "JEMPI_LINKER_MEMORY_RESERVE": "500M", + "JEMPI_PRE_PROCESSOR_MEMORY_LIMIT": "3G", + "JEMPI_PRE_PROCESSOR_MEMORY_RESERVE": "500M", + "JEMPI_SYNC_RECEIVER_MEMORY_LIMIT": "3G", + "JEMPI_SYNC_RECEIVER_MEMORY_RESERVE": "500M", + "JEMPI_API_MEMORY_LIMIT": "3G", + "JEMPI_API_MEMORY_RESERVE": "500M", "JEMPI_KAFKA_TOPICS": "JeMPI-async-preprocessor,JeMPI-patient-controller,JeMPI-patient-em,JeMPI-patient-linker,JeMPI-mu-linker,JeMPI-notifications", "JEMPI_KAFKA_DEBUG": false, - "ASYNC_RECEIVER_IMAGE_TAG": "0.1.0", - "SYNC_RECEIVER_IMAGE_TAG": "0.1.0", - "PRE_PROCESSOR_IMAGE_TAG": "0.1.0", - "CONTROLLER_IMAGE_TAG": "0.1.0", - "EM_CALCULATOR_IMAGE_TAG": "0.1.0", - "LINKER_IMAGE_TAG": "0.1.0", - "API_IMAGE_TAG": "0.1.0" + "JEMPI_AJEMPI_SYNC_RECEIVER_IMAGE_TAG": "0.1.0", + "JEMPI_SYNC_RECEIVER_IMAGE_TAG": "0.1.0", + "JEMPI_PRE_PROCESSOR_IMAGE_TAG": "0.1.0", + "JEMPI_CONTROLLER_IMAGE_TAG": "0.1.0", + "JEMPI_EM_CALCULATOR_IMAGE_TAG": "0.1.0", + "JEMPI_LINKER_IMAGE_TAG": "0.1.0", + "JEMPI_API_IMAGE_TAG": "0.1.0" } }