Implementation of Background Task functionality for #20 (#22)

* Kafka workers approach

* Initial work for background tasks

* Kafka setup updates

* Task runner and project cleanup

* CI fixes

.env

@@ -0,0 +1 @@
+POSTMAN_COLLECTION=

.github/workflows/main.yml

@@ -84,14 +84,14 @@ jobs:
           build-args: NODE_AUTH_TOKEN=${{ secrets.NPM_TOKEN }}
 
       - name: Pull containers
-        run: docker-compose -f ci.docker-compose.yml pull shr-fhir openhim-core mongo-db newman
+        run: docker-compose -f ci.docker-compose.yml pull shr-fhir openhim-core mongo-db newman kafka zookeeper
 
       - name: Cache containers
         uses: satackey/[email protected]
         continue-on-error: true
 
       - name: Start containers
-        run: docker-compose -f ci.docker-compose.yml up -d shr-fhir mongo-db openhim-core 
+        run: docker-compose -f ci.docker-compose.yml up -d shr-fhir mongo-db openhim-core kafka zookeeper
 
       - name: Load openhim config
         run: docker-compose -f ci.docker-compose.yml up -d openhim-config
@@ -106,13 +106,16 @@ jobs:
       - name: Run SHR image
         run: docker-compose -f ci.docker-compose.yml up -d shr
 
-      - name: Sleep for 10 seconds
-        run: sleep 10s
+      - name: Sleep for 30 seconds
+        run: sleep 30s
         shell: bash
 
       - name: Show SHR Log
         run: docker-compose -f "ci.docker-compose.yml" logs shr
 
+      - name: Show kafka Log
+        run: docker-compose -f "ci.docker-compose.yml" logs kafka
+
       - name: Check openhim-core
         run: curl -sSk https://localhost:8080/heartbeat
 
@@ -135,6 +138,12 @@ jobs:
           POSTMAN_COLLECTION: https://www.getpostman.com/collections/2ee8ebff39c078bac256
         run: docker-compose -f ci.docker-compose.yml up --exit-code-from newman newman
 
+      - name: Show SHR Log
+        run: docker-compose -f "ci.docker-compose.yml" logs shr
+
+      - name: Show kafka Log
+        run: docker-compose -f "ci.docker-compose.yml" logs kafka  
+
       - name: Stop containers
         if: always()
         run: docker-compose -f "dev.docker-compose.yml" down

.github/workflows/script/build-test.sh

@@ -1,17 +1,18 @@
 #!/bin/bash
-docker build ./ -t test-image:latest
-docker-compose -f ci.docker-compose.yml pull shr-fhir openhim-core mongo-db newman
-docker-compose -f ci.docker-compose.yml up -d shr-fhir mongo-db openhim-core 
+docker build ./ -t test-image:latest --build-args NODE_AUTH_TOKEN
+docker-compose -f ci.docker-compose.yml pull shr-fhir openhim-core mongo-db newman kafka zookeeper
+docker-compose -f ci.docker-compose.yml up -d shr-fhir mongo-db openhim-core kafka zookeeper
 
 sleep 60
 
 docker-compose -f ci.docker-compose.yml up -d openhim-config
 
 sleep 20
+
 docker-compose -f ci.docker-compose.yml logs openhim-config
 docker-compose -f ci.docker-compose.yml up -d shr
 
-docker-compose ci.docker-compose.yml ps
+docker-compose -f ci.docker-compose.yml ps
 
 declare -a tests=("https://www.getpostman.com/collections/481bb6cc8e1e964fd8bd" 
                 "https://www.getpostman.com/collections/ff5183adca5b5e720338" 
@@ -25,3 +26,5 @@ do
 
    docker-compose -f ci.docker-compose.yml up --exit-code-from newman newman 
 done
+
+# docker-compose -f ci.docker-compose.yml down -v

README.md

@@ -14,6 +14,9 @@ This project implements the following FHIR IGs:
 ```sh
 git clone https://github.com/I-TECH-UW/shared-health-record.git
 cd shared-health-record
+
+docker-compose -f .\dev.docker-compose.yml --profile dev up -d
+
 yarn
 yarn build
 yarn start

ci.docker-compose.yml

@@ -8,7 +8,7 @@ services:
     image: test-image:latest
     ports:
       - 3000:3000
-      - 2575:2575
+      - 3001:3001
     environment:
       - NODE_ENV=docker
     volumes:
@@ -57,7 +57,7 @@ services:
     container_name: openhim-config
     image: ghcr.io/i-tech-uw/openhim-config:latest
     volumes:
-      - ./config/openhim/test-openhim-config.json:/app/test-openhim-config.json
+      - ./config/openhim/ci-openhim-config.json:/app/test-openhim-config.json
 
   # Newman Tests
   newman:
@@ -66,7 +66,29 @@ services:
       - $POSTMAN_COLLECTION
     volumes:
       - ./.postman:/.postman
-    entrypoint: newman run $POSTMAN_COLLECTION -e /.postman/ci.postman_environment.json --insecure --timeout-request 10000
+    entrypoint: newman run $POSTMAN_COLLECTION -e /.postman/ci.postman_environment.json --insecure --timeout-request 10000 --delay-request 10000
+
+  # Task Runner
+  zookeeper:
+    image: 'bitnami/zookeeper:latest'
+    hostname: zookeeper
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+  kafka:
+    image: 'bitnami/kafka:latest'
+    hostname: kafka
+    container_name: kafka
+    environment:
+      - KAFKA_BROKER_ID=1
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
+      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
+      - ALLOW_PLAINTEXT_LISTENER=yes
+      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
+    volumes:
+      - kafka-data:/bitnami/kafka
+    depends_on:
+      - zookeeper
 
   # Not necessary
   openhim-console:
@@ -81,7 +103,8 @@ services:
       interval: 30s
       timeout: 30s
       retries: 3
-    
+
 volumes:
   mongo-data:
   hapi-data:
+  kafka-data:

config/config_ci.json

@@ -1,22 +1,23 @@
 {
   "app": {
     "port": 3000,
-    "mllpPort": 2575
+    "mllpPort": 3001
   },
   "mediator": {
     "api": {
       "username": "[email protected]",
-      "password": "openhim-password",
+      "password": "openhim",
       "apiURL": "https://openhim-core:8080",
       "trustSelfSigned": true,
       "urn": "urn:mediator:shared-health-record"
     }
   },
   "fhirServer": {
-    "mpiURL": "http://openhim-core:5000/CR/fhir",
-    "baseURL": "http://shr-fhir:8080/fhir",
-    "simulateResulting": false
+    "mpiURL": "https://openhim-core:5000/CR/fhir",
+    "baseURL": "http://shr-fhir:8080/fhir"
   },
-  "fhirConverterUrl": "http://localhost:5001/72f"
-
+  "fhirConverterUrl": "https://openhim-core:5000/72f",
+  "taskRunner": {
+    "brokers" : ["kafka:9092"]
+  }
 }

config/config_development.json

@@ -1,7 +1,7 @@
 {
   "app": {
     "port": 3000,
-    "mllpPort": 2575
+    "mllpPort": 3001
   },
   "mediator": {
     "api": {
@@ -17,9 +17,11 @@
     }
   },
   "fhirServer": {
-    "mpiURL": "https://localhost:5000/CR/fhir",
-    "baseURL": "https://localhost:8090/fhir", 
-    "simulateResulting": true
+    "mpiURL": "http://localhost:5001/CR/fhir",
+    "baseURL": "http://localhost:8090/fhir"
   },
-  "fhirConverterUrl": "http://localhost:5001/72f"
+  "fhirConverterUrl": "http://localhost:5001/72f",
+  "taskRunner": {
+    "workers" : ["localhost:9092"]
+  }
 }

config/config_test.json

@@ -1,7 +1,7 @@
 {
   "app": {
     "port": 3000,
-    "mllpPort": 2575
+    "mllpPort": 3001
   },
   "mediator": {
     "api": {

config/mediator.json

@@ -134,7 +134,7 @@
           "forwardAuthHeader": false,
           "name": "SHR - Laboratory",
           "secured": false,
-          "host": "host.docker.internal",
+          "host": "shr",
           "port": 3000,
           "path": "",
           "pathTransform": "s/SHR\\/fhir\\/ips/ips/g",

config/mediator_development.json

@@ -1,13 +1,13 @@
 {
   "urn": "urn:mediator:shared-health-record",
-  "version": "1.2.2-dev",
+  "version": "1.3.0-dev",
   "name": "Shared Health Record",
   "description": "Shared Health Record",
   "config": {
     "fhirServer": {
       "username": "",
       "password": "",
-      "baseURL": "https://localhost:8090/fhir"
+      "baseURL": "http://localhost:8090/fhir"
     },
     "clientRegistry": {
       "username": "",