From 3978ec30d0043485df51fd0d0979a00fa1d37cb4 Mon Sep 17 00:00:00 2001 From: "A. Anil Sinaci" Date: Fri, 20 Oct 2023 22:32:10 +0300 Subject: [PATCH] :recycle: refactor(log-service): Refactor server and log service for docker packaging. --- docker/docker-compose.yml | 21 ++--- docker/engine/Dockerfile-addJar | 2 +- docker/engine/Dockerfile-buildJar | 2 +- docker/engine/docker-entrypoint.sh | 4 +- docker/log-server/Dockerfile | 13 ++- docker/log-server/build.sh | 2 +- docker/log-server/docker-entrypoint.sh | 41 +++++++-- docker/server/Dockerfile | 13 ++- docker/server/build.sh | 2 +- docker/server/docker-entrypoint.sh | 83 ++++++++++++++++--- tofhir-engine/pom.xml | 2 +- .../scala/io/tofhir/engine/ToFhirEngine.scala | 1 + .../tofhir/engine/config/ToFhirConfig.scala | 1 + .../src/main/resources/application.conf | 2 +- .../tofhir/server/endpoint/JobEndpoint.scala | 2 +- .../server/endpoint/ProjectEndpoint.scala | 11 ++- .../src/main/resources/application.conf | 7 +- .../scala/io/tofhir/server/ToFhirServer.scala | 5 +- .../server/config/LogServiceConfig.scala | 10 +++ .../tofhir/server/endpoint/JobEndpoint.scala | 4 +- .../server/endpoint/ProjectEndpoint.scala | 5 +- .../endpoint/ToFhirServerEndpoint.scala | 6 +- .../server/service/ExecutionService.scala | 41 ++++----- .../io/tofhir/server/BaseEndpointTest.scala | 6 +- 24 files changed, 191 insertions(+), 95 deletions(-) create mode 100644 tofhir-server/src/main/scala/io/tofhir/server/config/LogServiceConfig.scala diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7493ed44..d6cacc6b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,32 +2,31 @@ version: '3.9' services: tofhir-log-server: image: srdc/tofhir-log-server:latest - container_name: tofhir-log - hostname: tofhir-log - environment: - - APP_CONF_FILE=/tofhir/conf/log-service.conf + container_name: tofhir-log-server + hostname: tofhir-log-server ports: - "8086:8086" networks: - tofhir-network volumes: - - './data-ingestion-suite:/tofhir/conf' - - './tofhir-docker-logs:/tofhir/logs' + - './tofhir-docker-logs:/usr/local/tofhir/logs' tofhir-server: image: srdc/tofhir-server:latest container_name: tofhir hostname: tofhir environment: - - LOG_SERVER_BASE_URL=http://tofhir-log:8086 + - LOG_SERVICE_ENDPOINT=http://tofhir-log-server:8086/tofhir-logs + - FHIR_DEFINITIONS_ENDPOINT=http://onfhir:8080/fhir - FHIR_REPO_URL=http://onfhir:8080/fhir - - APP_CONF_FILE=/tofhir/conf/application.conf + - APP_CONF_FILE=/usr/local/tofhir/conf/docker/tofhir-server.conf + - CONTEXT_PATH=conf ports: - "8085:8085" networks: - tofhir-network volumes: - - './data-ingestion-suite:/tofhir/conf' - - './tofhir-docker-logs:/tofhir/logs' + - './data-integration-suite:/usr/local/tofhir/conf' + - './tofhir-docker-logs:/usr/local/tofhir/logs' tofhir-web: image: srdc/tofhir-web:latest container_name: tofhir-web @@ -39,5 +38,3 @@ networks: tofhir-network: name: onfhir-network external: true -volumes: - fhirdb: {} diff --git a/docker/engine/Dockerfile-addJar b/docker/engine/Dockerfile-addJar index 15078032..08e0b7db 100644 --- a/docker/engine/Dockerfile-addJar +++ b/docker/engine/Dockerfile-addJar @@ -4,7 +4,7 @@ ENV TOFHIR_HOME /usr/local/tofhir RUN mkdir -p "$TOFHIR_HOME" WORKDIR $TOFHIR_HOME -COPY ./tofhir-engine/target/tofhir-standalone.jar . +COPY ./tofhir-engine/target/tofhir-engine-standalone.jar . COPY ./docker/engine/docker-entrypoint.sh . RUN chmod +x docker-entrypoint.sh diff --git a/docker/engine/Dockerfile-buildJar b/docker/engine/Dockerfile-buildJar index 5d8436cb..89b05e6b 100644 --- a/docker/engine/Dockerfile-buildJar +++ b/docker/engine/Dockerfile-buildJar @@ -17,7 +17,7 @@ ENV TOFHIR_HOME /usr/local/tofhir RUN mkdir -p "$TOFHIR_HOME" WORKDIR $TOFHIR_HOME -COPY --from=builder $TOFHIR_HOME/tofhir-engine/target/tofhir-standalone.jar . +COPY --from=builder $TOFHIR_HOME/tofhir-engine/target/tofhir-engine-standalone.jar . COPY docker/engine/docker-entrypoint.sh . RUN chmod +x docker-entrypoint.sh diff --git a/docker/engine/docker-entrypoint.sh b/docker/engine/docker-entrypoint.sh index 124ec614..f66b9471 100644 --- a/docker/engine/docker-entrypoint.sh +++ b/docker/engine/docker-entrypoint.sh @@ -37,7 +37,7 @@ if [ ! -z "$FHIR_BATCH_SIZE" ]; then JAVA_CMD+="-Dtofhir.fhir-writer.batch-group-size=$FHIR_BATCH_SIZE " fi if [ ! -z "$DB_PATH" ]; then - JAVA_CMD+="-Dtofhir.db=$DB_PATH " + JAVA_CMD+="-Dtofhir.db-path=$DB_PATH " fi # Delay the execution for this amount of seconds @@ -46,6 +46,6 @@ if [ ! -z "$DELAY_EXECUTION" ]; then fi # Finally, tell which jar to run -JAVA_CMD+="tofhir-standalone.jar" +JAVA_CMD+="tofhir-engine-standalone.jar" eval $JAVA_CMD "$@" diff --git a/docker/log-server/Dockerfile b/docker/log-server/Dockerfile index d5065e8b..9095e30c 100644 --- a/docker/log-server/Dockerfile +++ b/docker/log-server/Dockerfile @@ -1,14 +1,11 @@ FROM openjdk:11-jre-slim -# Create folders -RUN mkdir -p /tofhir -WORKDIR /tofhir +ENV TOFHIR_HOME /usr/local/tofhir +RUN mkdir -p "$TOFHIR_HOME" +WORKDIR $TOFHIR_HOME -# Add Files COPY ./tofhir-log-server/target/tofhir-log-server-standalone.jar . COPY ./docker/log-server/docker-entrypoint.sh . -COPY ./tofhir-log-server/src/main/resources/logback.xml . +RUN chmod +x docker-entrypoint.sh -RUN chmod +x /tofhir/docker-entrypoint.sh - -ENTRYPOINT ["/tofhir/docker-entrypoint.sh"] +ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/docker/log-server/build.sh b/docker/log-server/build.sh index 2caa798b..196a84d0 100644 --- a/docker/log-server/build.sh +++ b/docker/log-server/build.sh @@ -1,4 +1,4 @@ # Execute one of the following commands from the project.root.directory (../../) -docker build -f docker/log-server/Dockerfile -t srdc/tofhir-log-server:latest -t srdc/tofhir-log-server:1.0 . +docker build -f docker/log-server/Dockerfile -t srdc/tofhir-log-server:latest . diff --git a/docker/log-server/docker-entrypoint.sh b/docker/log-server/docker-entrypoint.sh index 703e5c0f..1ddcebca 100644 --- a/docker/log-server/docker-entrypoint.sh +++ b/docker/log-server/docker-entrypoint.sh @@ -1,15 +1,42 @@ -#!/bin/bash +#!/usr/bin/env bash -JAVA_CMD="java -jar " +JAVA_CMD="java -Xms256m -Xmx3g -jar " # Configure application.conf path if [ ! -z "$APP_CONF_FILE" ]; then - JAVA_CMD+="-Dconfig.file=$APP_CONF_FILE " + JAVA_CMD+="-Dconfig.file=$APP_CONF_FILE " fi -# Finally, tell which jar to run -JAVA_CMD+="/tofhir/tofhir-log-server-standalone.jar" +# Configure logback configuration file +if [ ! -z "$LOGBACK_CONF_FILE" ]; then + JAVA_CMD+="-Dlogback.configurationFile=$LOGBACK_CONF_FILE " +fi + +# Configure Spark +if [ ! -z "$SPARK_APPNAME" ]; then + JAVA_CMD+="-Dspark.app-name=$SPARK_APPNAME " +fi +if [ ! -z "$SPARK_MASTER" ]; then + JAVA_CMD+="-Dspark.master=$SPARK_MASTER " +fi + +# Configure tofhir-server web server +if [ ! -z "$WEBSERVER_HOST" ]; then + JAVA_CMD+="-Dwebserver.host=$WEBSERVER_HOST " +fi +if [ ! -z "$WEBSERVER_PORT" ]; then + JAVA_CMD+="-Dwebserver.port=$WEBSERVER_PORT " +fi +if [ ! -z "$WEBSERVER_BASEURI" ]; then + JAVA_CMD+="-Dwebserver.base-uri=$WEBSERVER_BASEURI " +fi -echo "Running command: $JAVA_CMD" +# Delay the execution for this amount of seconds +if [ ! -z "$DELAY_EXECUTION" ]; then + sleep $DELAY_EXECUTION +fi + +# Finally, tell which jar to run +JAVA_CMD+="tofhir-log-server-standalone.jar" -eval $JAVA_CMD +eval $JAVA_CMD "$@" diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 464d4f97..4fe79a22 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,14 +1,11 @@ FROM openjdk:11-jre-slim -# Create folders -RUN mkdir -p /tofhir -WORKDIR /tofhir +ENV TOFHIR_HOME /usr/local/tofhir +RUN mkdir -p "$TOFHIR_HOME" +WORKDIR $TOFHIR_HOME -# Add Files COPY ./tofhir-server/target/tofhir-server-standalone.jar . COPY ./docker/server/docker-entrypoint.sh . -COPY ./tofhir-server/src/main/resources/logback.xml . +RUN chmod +x docker-entrypoint.sh -RUN chmod +x /tofhir/docker-entrypoint.sh - -ENTRYPOINT ["/tofhir/docker-entrypoint.sh"] +ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/docker/server/build.sh b/docker/server/build.sh index d7696211..e0ccc5bf 100644 --- a/docker/server/build.sh +++ b/docker/server/build.sh @@ -1,4 +1,4 @@ # Execute one of the following commands from the project.root.directory (../../) -docker build -f docker/server/Dockerfile -t srdc/tofhir-server:latest -t srdc/tofhir-server:1.0 . +docker build -f docker/server/Dockerfile -t srdc/tofhir-server:latest . diff --git a/docker/server/docker-entrypoint.sh b/docker/server/docker-entrypoint.sh index e7eb091a..a380f956 100644 --- a/docker/server/docker-entrypoint.sh +++ b/docker/server/docker-entrypoint.sh @@ -1,23 +1,84 @@ -#!/bin/bash +#!/usr/bin/env bash -JAVA_CMD="java -jar " +JAVA_CMD="java -Xms256m -Xmx3g -jar " # Configure application.conf path if [ ! -z "$APP_CONF_FILE" ]; then - JAVA_CMD+="-Dconfig.file=$APP_CONF_FILE " + JAVA_CMD+="-Dconfig.file=$APP_CONF_FILE " fi -if [ ! -z "$FHIR_REPO_URL" ]; then - JAVA_CMD+="-Dfhir.definitions-fhir-endpoint=$FHIR_REPO_URL " +# Configure logback configuration file +if [ ! -z "$LOGBACK_CONF_FILE" ]; then + JAVA_CMD+="-Dlogback.configurationFile=$LOGBACK_CONF_FILE " fi -if [ ! -z "$LOG_SERVER_BASE_URL" ]; then - JAVA_CMD+="-webserver.log-server-base-url=$LOG_SERVER_BASE_URL " +# Configure Spark +if [ ! -z "$SPARK_APPNAME" ]; then + JAVA_CMD+="-Dspark.app-name=$SPARK_APPNAME " +fi +if [ ! -z "$SPARK_MASTER" ]; then + JAVA_CMD+="-Dspark.master=$SPARK_MASTER " fi -# Finally, tell which jar to run -JAVA_CMD+="/tofhir/tofhir-server-standalone.jar" +# Configure toFHIR mapping-related paths +if [ ! -z "$CONTEXT_PATH" ]; then + JAVA_CMD+="-Dtofhir.context-path=$CONTEXT_PATH " +fi +if [ ! -z "$MAPPINGS_FOLDER" ]; then + JAVA_CMD+="-Dtofhir.mappings.repository.folder-path=$MAPPINGS_FOLDER " +fi +if [ ! -z "$SCHEMAS_FOLDER" ]; then + JAVA_CMD+="-Dtofhir.mappings.schemas.repository.folder-path=$SCHEMAS_FOLDER " +fi +if [ ! -z "$MAPPING_JOB" ]; then + JAVA_CMD+="-Dtofhir.mapping-jobs.initial-job-file-path=$MAPPING_JOB " +fi +if [ ! -z "$FHIR_BATCH_SIZE" ]; then + JAVA_CMD+="-Dtofhir.fhir-writer.batch-group-size=$FHIR_BATCH_SIZE " +fi +if [ ! -z "$DB_PATH" ]; then + JAVA_CMD+="-Dtofhir.db-path=$DB_PATH " +fi -echo "Running command: $JAVA_CMD" +# Configure the FHIR endpoint to which this toFHIR can connect to retrieve resource definitions etc. +if [ ! -z "$FHIR_DEFINITIONS_ENDPOINT" ]; then + JAVA_CMD+="-Dfhir.definitions-fhir-endpoint=$FHIR_DEFINITIONS_ENDPOINT " +fi +if [ ! -z "$FHIR_DEFINITIONS_ROOT_URL" ]; then + JAVA_CMD+="-Dfhir.definitions-root-urls=$FHIR_DEFINITIONS_ROOT_URL " +fi +if [ ! -z "$FHIR_DEFINITIONS_PROFILES_PATH" ]; then + JAVA_CMD+="-Dfhir.profiles-path=$FHIR_DEFINITIONS_ROOT_URL " +fi +if [ ! -z "$FHIR_DEFINITIONS_VALUESETS_PATH" ]; then + JAVA_CMD+="-Dfhir.valuesets-path=$FHIR_DEFINITIONS_VALUESETS_PATH " +fi +if [ ! -z "$FHIR_DEFINITIONS_CODESYSTEMS_URL" ]; then + JAVA_CMD+="-Dfhir.codesystems-path=$FHIR_DEFINITIONS_CODESYSTEMS_URL " +fi + +# Configure tofhir-server web server +if [ ! -z "$WEBSERVER_HOST" ]; then + JAVA_CMD+="-Dwebserver.host=$WEBSERVER_HOST " +fi +if [ ! -z "$WEBSERVER_PORT" ]; then + JAVA_CMD+="-Dwebserver.port=$WEBSERVER_PORT " +fi +if [ ! -z "$WEBSERVER_BASEURI" ]; then + JAVA_CMD+="-Dwebserver.base-uri=$WEBSERVER_BASEURI " +fi + +# Configure log service +if [ ! -z "$LOG_SERVICE_ENDPOINT" ]; then + JAVA_CMD+="-Dlog-service.endpoint=$LOG_SERVICE_ENDPOINT " +fi + +# Delay the execution for this amount of seconds +if [ ! -z "$DELAY_EXECUTION" ]; then + sleep $DELAY_EXECUTION +fi + +# Finally, tell which jar to run +JAVA_CMD+="tofhir-server-standalone.jar" -eval $JAVA_CMD +eval $JAVA_CMD "$@" diff --git a/tofhir-engine/pom.xml b/tofhir-engine/pom.xml index fe9b5135..bf594e3a 100644 --- a/tofhir-engine/pom.xml +++ b/tofhir-engine/pom.xml @@ -92,7 +92,7 @@ false - tofhir-standalone + tofhir-engine-standalone - { - request.projectId = Some(projectId) - jobEndpoint.route(request) - } - } + pathPrefix(Segment) { projectId: String => { + request.projectId = Some(projectId) + jobEndpoint.route(request) + } + } } } } diff --git a/tofhir-server/src/main/resources/application.conf b/tofhir-server/src/main/resources/application.conf index 39f54bbe..48644566 100644 --- a/tofhir-server/src/main/resources/application.conf +++ b/tofhir-server/src/main/resources/application.conf @@ -108,8 +108,6 @@ webserver = { # Base Uri for server e.g. With this default configuration, the root path of toFHIR server will be http://localhost:8085/tofhir base-uri = tofhir - log-server-base-url = "http://localhost:8086" - ssl { # Path to the java keystore for enabling ssl for toFHIR server, use null to disable ssl keystore = null @@ -118,6 +116,11 @@ webserver = { } } +# The service from where tofhir-server will read the logs. +log-service = { + endpoint = "http://localhost:8086/tofhir-logs" +} + # Spark configurations spark = { app.name = "AICCELERATE Data Integration Suite" diff --git a/tofhir-server/src/main/scala/io/tofhir/server/ToFhirServer.scala b/tofhir-server/src/main/scala/io/tofhir/server/ToFhirServer.scala index 2878c311..c4fbf3ce 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/ToFhirServer.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/ToFhirServer.scala @@ -1,7 +1,7 @@ package io.tofhir.server import io.tofhir.engine.config.ToFhirConfig -import io.tofhir.server.config.WebServerConfig +import io.tofhir.server.config.{LogServiceConfig, WebServerConfig} import io.tofhir.server.endpoint.ToFhirServerEndpoint import io.tofhir.server.fhir.FhirDefinitionsConfig @@ -11,7 +11,8 @@ object ToFhirServer { val webServerConfig = new WebServerConfig(actorSystem.settings.config.getConfig("webserver")) val fhirDefinitionsConfig = new FhirDefinitionsConfig(actorSystem.settings.config.getConfig("fhir")) - val endpoint = new ToFhirServerEndpoint(ToFhirConfig.engineConfig, webServerConfig, fhirDefinitionsConfig) + val logServiceConfig = new LogServiceConfig(actorSystem.settings.config.getConfig("log-service")) + val endpoint = new ToFhirServerEndpoint(ToFhirConfig.engineConfig, webServerConfig, fhirDefinitionsConfig, logServiceConfig) ToFhirHttpServer.start(endpoint.toFHIRRoute, webServerConfig) } diff --git a/tofhir-server/src/main/scala/io/tofhir/server/config/LogServiceConfig.scala b/tofhir-server/src/main/scala/io/tofhir/server/config/LogServiceConfig.scala new file mode 100644 index 00000000..5c736467 --- /dev/null +++ b/tofhir-server/src/main/scala/io/tofhir/server/config/LogServiceConfig.scala @@ -0,0 +1,10 @@ +package io.tofhir.server.config + +import com.typesafe.config.Config + +import scala.util.Try + +class LogServiceConfig(logServiceConfig: Config) { + /** Host name/address to start service on. */ + lazy val logServiceEndpoint: String = Try(logServiceConfig.getString("endpoint")).getOrElse("http://localhost:8086/tofhir-log") +} diff --git a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/JobEndpoint.scala b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/JobEndpoint.scala index b8ac9236..33f53016 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/JobEndpoint.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/JobEndpoint.scala @@ -17,10 +17,10 @@ import io.tofhir.server.service.job.IJobRepository import io.tofhir.server.service.mapping.IMappingRepository import io.tofhir.server.service.schema.ISchemaRepository -class JobEndpoint(jobRepository: IJobRepository, mappingRepository: IMappingRepository, schemaRepository: ISchemaRepository) extends LazyLogging { +class JobEndpoint(jobRepository: IJobRepository, mappingRepository: IMappingRepository, schemaRepository: ISchemaRepository, logServiceEndpoint: String) extends LazyLogging { val service: JobService = new JobService(jobRepository) - val executionService: ExecutionService = new ExecutionService(jobRepository, mappingRepository, schemaRepository) + val executionService: ExecutionService = new ExecutionService(jobRepository, mappingRepository, schemaRepository, logServiceEndpoint) def route(request: ToFhirRestCall): Route = { pathPrefix(SEGMENT_JOB) { diff --git a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ProjectEndpoint.scala b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ProjectEndpoint.scala index 9dc0a522..a100411d 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ProjectEndpoint.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ProjectEndpoint.scala @@ -25,12 +25,13 @@ class ProjectEndpoint(schemaRepository: ISchemaRepository, mappingRepository: IMappingRepository, jobRepository: IJobRepository, mappingContextRepository: IMappingContextRepository, - projectRepository: IProjectRepository) extends LazyLogging { + projectRepository: IProjectRepository, + logServiceEndpoint: String) extends LazyLogging { val service: ProjectService = new ProjectService(projectRepository) val schemaDefinitionEndpoint: SchemaDefinitionEndpoint = new SchemaDefinitionEndpoint(schemaRepository, mappingRepository) val mappingEndpoint: MappingEndpoint = new MappingEndpoint(mappingRepository) - val jobEndpoint: JobEndpoint = new JobEndpoint(jobRepository, mappingRepository, schemaRepository) + val jobEndpoint: JobEndpoint = new JobEndpoint(jobRepository, mappingRepository, schemaRepository, logServiceEndpoint) val mappingContextEndpoint: MappingContextEndpoint = new MappingContextEndpoint(mappingContextRepository) def route(request: ToFhirRestCall): Route = { diff --git a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ToFhirServerEndpoint.scala b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ToFhirServerEndpoint.scala index 46728018..e5c32a2f 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ToFhirServerEndpoint.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ToFhirServerEndpoint.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.{HttpMethod, Uri} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.{RejectionHandler, Route} import io.tofhir.engine.config.ToFhirEngineConfig -import io.tofhir.server.config.WebServerConfig +import io.tofhir.server.config.{LogServiceConfig, WebServerConfig} import io.tofhir.server.fhir.FhirDefinitionsConfig import io.tofhir.server.interceptor.{ICORSHandler, IErrorHandler} import io.tofhir.server.model.ToFhirRestCall @@ -21,7 +21,7 @@ import java.util.UUID * Encapsulates all services and directives * Main Endpoint for toFHIR server */ -class ToFhirServerEndpoint(toFhirEngineConfig: ToFhirEngineConfig, webServerConfig: WebServerConfig, fhirDefinitionsConfig: FhirDefinitionsConfig) extends ICORSHandler with IErrorHandler { +class ToFhirServerEndpoint(toFhirEngineConfig: ToFhirEngineConfig, webServerConfig: WebServerConfig, fhirDefinitionsConfig: FhirDefinitionsConfig, logServiceConfig: LogServiceConfig) extends ICORSHandler with IErrorHandler { val projectRepository: ProjectFolderRepository = new ProjectFolderRepository(toFhirEngineConfig) // creating the repository instance globally as weed a singleton instance val mappingRepository: ProjectMappingFolderRepository = new ProjectMappingFolderRepository(toFhirEngineConfig.mappingRepositoryFolderPath, projectRepository) @@ -32,7 +32,7 @@ class ToFhirServerEndpoint(toFhirEngineConfig: ToFhirEngineConfig, webServerConf // Initialize the projects by reading the resources available in the file system new FolderDBInitializer(toFhirEngineConfig, schemaRepository, mappingRepository, mappingJobRepository, projectRepository, mappingContextRepository).init() - val projectEndpoint = new ProjectEndpoint(schemaRepository, mappingRepository, mappingJobRepository, mappingContextRepository, projectRepository) + val projectEndpoint = new ProjectEndpoint(schemaRepository, mappingRepository, mappingJobRepository, mappingContextRepository, projectRepository, logServiceConfig.logServiceEndpoint) val fhirDefinitionsEndpoint = new FhirDefinitionsEndpoint(fhirDefinitionsConfig) val fhirPathFunctionsEndpoint = new FhirPathFunctionsEndpoint() val terminologyServiceManagerEndpoint = new TerminologyServiceManagerEndpoint(terminologySystemFolderRepository, mappingJobRepository, toFhirEngineConfig) diff --git a/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala b/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala index eff53886..b92ca117 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala @@ -1,12 +1,12 @@ package io.tofhir.server.service -import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.model._ import com.typesafe.scalalogging.LazyLogging import io.onfhir.path.IFhirPathFunctionLibraryFactory import io.tofhir.common.util.CustomMappingFunctionsFactory import io.tofhir.engine.ToFhirEngine +import io.tofhir.engine.Execution import io.tofhir.engine.config.ErrorHandlingType.ErrorHandlingType import io.tofhir.engine.config.ToFhirConfig import io.tofhir.engine.mapping.{FhirMappingJobManager, MappingContextLoader} @@ -15,12 +15,12 @@ import io.tofhir.engine.util.FhirMappingJobFormatter.formats import io.tofhir.engine.util.FileUtils import io.tofhir.engine.util.FileUtils.FileExtensions import io.tofhir.rxnorm.RxNormApiFunctionLibraryFactory +import io.tofhir.server.interceptor.ICORSHandler import io.tofhir.server.model.{BadRequest, ExecuteJobTask, ResourceNotFound, TestResourceCreationRequest} import io.tofhir.server.service.job.IJobRepository import io.tofhir.server.service.mapping.IMappingRepository import io.tofhir.server.service.schema.ISchemaRepository import io.tofhir.server.util.DataFrameUtil -import io.tofhir.server.interceptor.ICORSHandler import org.apache.commons.io import org.json4s.JsonAST.{JBool, JObject, JValue} import org.json4s.JsonDSL.jobject2assoc @@ -30,7 +30,7 @@ import org.json4s.{JArray, JString} import java.io.File import java.util.UUID import scala.concurrent.duration.DurationInt -import scala.concurrent.{Await, ExecutionContextExecutor, Future} +import scala.concurrent.{Await, ExecutionContext, Future} /** * Service to handle all execution related operations @@ -39,8 +39,9 @@ import scala.concurrent.{Await, ExecutionContextExecutor, Future} * @param jobRepository * @param mappingRepository * @param schemaRepository + * @param logServiceEndpoint */ -class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappingRepository, schemaRepository: ISchemaRepository) extends LazyLogging { +class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappingRepository, schemaRepository: ISchemaRepository, logServiceEndpoint: String) extends LazyLogging { val externalMappingFunctions: Map[String, IFhirPathFunctionLibraryFactory] = Map( "rxn" -> new RxNormApiFunctionLibraryFactory("https://rxnav.nlm.nih.gov", 2), @@ -48,10 +49,9 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin ) val toFhirEngine = new ToFhirEngine(Some(mappingRepository), Some(schemaRepository), externalMappingFunctions) - implicit val system: ActorSystem = ActorSystem("akka-http-client") - implicit val executionContext: ExecutionContextExecutor = system.dispatcher + import Execution.actorSystem + implicit val ec: ExecutionContext = actorSystem.dispatcher - val logServerBaseUrl: String = system.settings.config.getConfig("webserver").getString("log-server-base-url") /** * Run the job for the given execute job tasks * @@ -73,7 +73,7 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin case None => mappingJob.mappings } - if(mappingTasks.isEmpty) { + if (mappingTasks.isEmpty) { throw BadRequest("No mapping task to execute!", "No mapping task to execute!") } @@ -205,7 +205,7 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin Future { val request = HttpRequest( method = HttpMethods.GET, - uri = s"$logServerBaseUrl/tofhir/projects/$projectId/jobs/$jobId/executions/$executionId/logs" + uri = s"$logServiceEndpoint/projects/$projectId/jobs/$jobId/executions/$executionId/logs" ) val responseFuture: Future[HttpResponse] = Http().singleRequest(request) @@ -215,14 +215,15 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin responseFuture .flatMap { resp => { resp.entity.toStrict(timeout) - }} + } + } .map { strictEntity => strictEntity.data.utf8String }, timeout ) val mappingTasksLogsResponse: Seq[JValue] = JsonMethods.parse(responseAsString).extract[Seq[JValue]] - mappingTasksLogsResponse.map(logResponse =>{ + mappingTasksLogsResponse.map(logResponse => { val logResponseObject: JObject = logResponse.asInstanceOf[JObject] JObject( @@ -253,7 +254,7 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin val page = queryParams.getOrElse("page", "1").toInt val request = HttpRequest( method = HttpMethods.GET, - uri = s"$logServerBaseUrl/tofhir/projects/$projectId/jobs/$jobId/executions?page=$page" + uri = s"$logServiceEndpoint/projects/$projectId/jobs/$jobId/executions?page=$page" ) val responseFuture: Future[HttpResponse] = Http().singleRequest(request) @@ -264,7 +265,8 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin .flatMap { resp => { countHeader = resp.headers.find(_.name == ICORSHandler.X_TOTAL_COUNT_HEADER).map(_.value).get.toInt resp.entity.toStrict(timeout) - }} + } + } .map { strictEntity => strictEntity.data.utf8String }, timeout ) @@ -272,11 +274,10 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin val paginatedLogsResponse: Seq[JValue] = JsonMethods.parse(responseAsString).extract[Seq[JValue]] - // Retrieve the running executions for the given job val jobExecutions: Set[String] = toFhirEngine.runningJobRegistry.getRunningExecutions(jobId) - val ret = paginatedLogsResponse.map(log =>{ + val ret = paginatedLogsResponse.map(log => { val logJson: JObject = log.asInstanceOf[JObject] JObject( logJson.obj :+ ("runningStatus" -> JBool(jobExecutions.contains((logJson \ "id").extract[String]))) @@ -291,9 +292,9 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin /** * Returns the execution logs for a specific execution ID. * - * @param projectId project id the job belongs to - * @param jobId job id - * @param executionId execution id + * @param projectId project id the job belongs to + * @param jobId job id + * @param executionId execution id * @return the execution summary as a JSON object */ def getExecutionById(projectId: String, jobId: String, executionId: String): Future[JObject] = { @@ -302,14 +303,14 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin case Some(_) => val request = HttpRequest( method = HttpMethods.GET, - uri = s"$logServerBaseUrl/tofhir/projects/$projectId/jobs/$jobId/executions/$executionId/logs" + uri = s"$logServiceEndpoint/projects/$projectId/jobs/$jobId/executions/$executionId/logs" ) val responseFuture: Future[HttpResponse] = Http().singleRequest(request) val timeout = 20000.millis val responseAsString = Await.result( responseFuture - .flatMap { resp => resp.entity.toStrict(timeout)} + .flatMap { resp => resp.entity.toStrict(timeout) } .map { strictEntity => strictEntity.data.utf8String }, timeout ) diff --git a/tofhir-server/src/test/scala/io/tofhir/server/BaseEndpointTest.scala b/tofhir-server/src/test/scala/io/tofhir/server/BaseEndpointTest.scala index d914d755..b9083d2e 100644 --- a/tofhir-server/src/test/scala/io/tofhir/server/BaseEndpointTest.scala +++ b/tofhir-server/src/test/scala/io/tofhir/server/BaseEndpointTest.scala @@ -6,11 +6,10 @@ import akka.http.scaladsl.testkit.ScalatestRouteTest import io.onfhir.util.JsonFormatter.formats import io.tofhir.engine.config.ToFhirEngineConfig import io.tofhir.engine.util.FileUtils -import io.tofhir.server.config.WebServerConfig +import io.tofhir.server.config.{LogServiceConfig, WebServerConfig} import io.tofhir.server.endpoint.ToFhirServerEndpoint import io.tofhir.server.fhir.FhirDefinitionsConfig import io.tofhir.server.model.Project -import io.tofhir.server.service.terminology.TerminologySystemFolderRepository import org.json4s.jackson.JsonMethods import org.json4s.jackson.Serialization.writePretty import org.scalatest.BeforeAndAfterAll @@ -24,6 +23,7 @@ trait BaseEndpointTest extends AnyWordSpec with Matchers with ScalatestRouteTest val toFhirEngineConfig: ToFhirEngineConfig = new ToFhirEngineConfig(system.settings.config.getConfig("tofhir")) val webServerConfig = new WebServerConfig(system.settings.config.getConfig("webserver")) val fhirDefinitionsConfig = new FhirDefinitionsConfig(system.settings.config.getConfig("fhir")) + val logServiceConfig = new LogServiceConfig(system.settings.config.getConfig("log-service")) // route endpoint var route: Route = _ @@ -63,7 +63,7 @@ trait BaseEndpointTest extends AnyWordSpec with Matchers with ScalatestRouteTest FileUtils.getPath(fhirDefinitionsConfig.codesystemsPath.get).toFile.mkdirs() FileUtils.getPath(fhirDefinitionsConfig.valuesetsPath.get).toFile.mkdirs() // initialize endpoint and route - val endpoint = new ToFhirServerEndpoint(toFhirEngineConfig, webServerConfig, fhirDefinitionsConfig) + val endpoint = new ToFhirServerEndpoint(toFhirEngineConfig, webServerConfig, fhirDefinitionsConfig, logServiceConfig) route = endpoint.toFHIRRoute }