diff --git a/tofhir-common/src/main/scala/io/tofhir/common/env/EnvironmentVariableResolver.scala b/tofhir-common/src/main/scala/io/tofhir/common/env/EnvironmentVariableResolver.scala deleted file mode 100644 index 85db6083..00000000 --- a/tofhir-common/src/main/scala/io/tofhir/common/env/EnvironmentVariableResolver.scala +++ /dev/null @@ -1,15 +0,0 @@ -package io.tofhir.common.env - -object EnvironmentVariableResolver { - - def replaceEnvironmentVariables(fileContent: String): String = { - var returningContent = fileContent; - // val regex = """\$\{(.*?)\}""".r - EnvironmentVariable.values.foreach { e => - val regex = "\\$\\{" + e.toString + "\\}" - if (sys.env.contains(e.toString)) returningContent = returningContent.replaceAll(regex, sys.env(e.toString)) - } - returningContent - } - -} diff --git a/tofhir-common/src/main/scala/io/tofhir/common/env/EnvironmentVariable.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/env/EnvironmentVariable.scala similarity index 86% rename from tofhir-common/src/main/scala/io/tofhir/common/env/EnvironmentVariable.scala rename to tofhir-engine/src/main/scala/io/tofhir/engine/env/EnvironmentVariable.scala index 2cc85e00..70b7c684 100644 --- a/tofhir-common/src/main/scala/io/tofhir/common/env/EnvironmentVariable.scala +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/env/EnvironmentVariable.scala @@ -1,4 +1,4 @@ -package io.tofhir.common.env +package io.tofhir.engine.env object EnvironmentVariable extends Enumeration { type EnvironmentVariable = Value diff --git a/tofhir-engine/src/main/scala/io/tofhir/engine/env/EnvironmentVariableResolver.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/env/EnvironmentVariableResolver.scala new file mode 100644 index 00000000..e03bf208 --- /dev/null +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/env/EnvironmentVariableResolver.scala @@ -0,0 +1,110 @@ +package io.tofhir.engine.env + +import io.tofhir.engine.model.{FhirMappingJob, FhirRepositorySinkSettings, FhirSinkSettings, FileSystemSourceSettings, MappingJobSourceSettings} + +/** + * Provide functions to find and resolve the environment variables used within FhirMappingJob definitions. + * An ENV_VAR is given as ${ENV_VAR} within the MappingJob definition. + */ +object EnvironmentVariableResolver { + + /** + * Reads the environment variables defined in the EnvironmentVariable object. + * + * @return A Map[String, String] containing the environment variable names and their values. + */ + def getEnvironmentVariables: Map[String, String] = { + EnvironmentVariable.values + .map(_.toString) // Convert Enumeration values to String (names of the variables) + .flatMap { envVar => + sys.env.get(envVar).map(envVar -> _) // Get the environment variable value if it exists + } + .toMap + } + + /** + * Resolves the environment variables from a given String which is a file content of a FhirMappingJob definition. + * + * @param fileContent The file content potentially containing placeholders for environment variables. + * @return The file content with all recognized environment variables resolved. + */ + def resolveFileContent(fileContent: String): String = { + EnvironmentVariable.values.foldLeft(fileContent) { (updatedContent, envVar) => + val placeholder = "\\$\\{" + envVar.toString + "\\}" + sys.env.get(envVar.toString) match { + case Some(envValue) => + updatedContent.replaceAll(placeholder, envValue) + case None => + updatedContent // Leave the content unchanged if the environment variable is not set + } + } + } + + /** + * Resolves all environment variable placeholders in a FhirMappingJob instance. + * + * @param job FhirMappingJob instance. + * @return A new FhirMappingJob with resolved values. + */ + def resolveFhirMappingJob(job: FhirMappingJob): FhirMappingJob = { + job.copy( + sourceSettings = resolveSourceSettings(job.sourceSettings), + sinkSettings = resolveSinkSettings(job.sinkSettings) + ) + } + + /** + * Resolves environment variables in MappingJobSourceSettings. + * + * @param sourceSettings A map of source settings. + * @return The map with resolved settings. + */ + private def resolveSourceSettings(sourceSettings: Map[String, MappingJobSourceSettings]): Map[String, MappingJobSourceSettings] = { + sourceSettings.map { case (key, value) => + key -> (value match { + case fs: FileSystemSourceSettings => + fs.copy(dataFolderPath = resolveEnvironmentVariables(fs.dataFolderPath)) + case other => other // No changes for other types + }) + } + } + + + /** + * Resolves environment variables in FhirRepositorySinkSettings. + * + * @param sinkSettings FhirSinkSettings instance. + * @return FhirSinkSettings with resolved fhirRepoUrl, if applicable. + */ + private def resolveSinkSettings(sinkSettings: FhirSinkSettings): FhirSinkSettings = { + sinkSettings match { + case fr: FhirRepositorySinkSettings => + fr.copy(fhirRepoUrl = resolveEnvironmentVariables(fr.fhirRepoUrl)) + case other => other // No changes for other types + } + } + + /** + * Resolves placeholders in the format ${ENV_VAR} with their corresponding values + * from the system environment variables, but only if the variables are part + * of the EnvironmentVariable enumeration. + * + * @param value The string potentially containing placeholders. + * @return The resolved string with placeholders replaced. + */ + private def resolveEnvironmentVariables(value: String): String = { + val pattern = """\$\{([A-Z_]+)\}""".r + + pattern.replaceAllIn(value, m => { + val envVar = m.group(1) + if (EnvironmentVariable.values.exists(_.toString == envVar)) { + sys.env.getOrElse(envVar, { + throw new RuntimeException(s"Environment variable $envVar is not set.") + }) + } else { + throw new RuntimeException(s"Environment variable $envVar is not recognized in EnvironmentVariable enumeration.") + } + }) + } + +} diff --git a/tofhir-common/src/main/scala/io/tofhir/common/model/ICachedRepository.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/repository/ICachedRepository.scala similarity index 84% rename from tofhir-common/src/main/scala/io/tofhir/common/model/ICachedRepository.scala rename to tofhir-engine/src/main/scala/io/tofhir/engine/repository/ICachedRepository.scala index 154e8701..7d5f2a3b 100644 --- a/tofhir-common/src/main/scala/io/tofhir/common/model/ICachedRepository.scala +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/repository/ICachedRepository.scala @@ -1,4 +1,4 @@ -package io.tofhir.common.model +package io.tofhir.engine.repository /** * A cached repository. diff --git a/tofhir-engine/src/main/scala/io/tofhir/engine/repository/mapping/IFhirMappingRepository.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/repository/mapping/IFhirMappingRepository.scala index bb289a33..1450ba66 100644 --- a/tofhir-engine/src/main/scala/io/tofhir/engine/repository/mapping/IFhirMappingRepository.scala +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/repository/mapping/IFhirMappingRepository.scala @@ -1,7 +1,7 @@ package io.tofhir.engine.repository.mapping -import io.tofhir.common.model.ICachedRepository import io.tofhir.engine.model.FhirMapping +import io.tofhir.engine.repository.ICachedRepository trait IFhirMappingRepository extends ICachedRepository { /** diff --git a/tofhir-engine/src/main/scala/io/tofhir/engine/util/FhirMappingJobFormatter.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/util/FhirMappingJobFormatter.scala index 7ca49d18..a270eb6a 100644 --- a/tofhir-engine/src/main/scala/io/tofhir/engine/util/FhirMappingJobFormatter.scala +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/util/FhirMappingJobFormatter.scala @@ -1,10 +1,10 @@ package io.tofhir.engine.util import io.onfhir.client.model.{BasicAuthenticationSettings, BearerTokenAuthorizationSettings, FixedTokenAuthenticationSettings} -import io.tofhir.common.env.EnvironmentVariableResolver -import io.tofhir.engine.model.{FhirMappingJob, FhirMappingTask, FhirRepositorySinkSettings, FhirServerSource, FhirServerSourceSettings, FileSystemSinkSettings, FileSystemSource, FileSystemSourceSettings, KafkaSource, KafkaSourceSettings, LocalFhirTerminologyServiceSettings, SQLSchedulingSettings, SchedulingSettings, SqlSource, SqlSourceSettings} -import org.json4s.{Formats, MappingException, ShortTypeHints} +import io.tofhir.engine.env.EnvironmentVariableResolver +import io.tofhir.engine.model._ import org.json4s.jackson.Serialization +import org.json4s.{Formats, MappingException, ShortTypeHints} import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} @@ -60,7 +60,7 @@ object FhirMappingJobFormatter { */ def readMappingJobFromFile(filePath: String): FhirMappingJob = { val source = Source.fromFile(filePath, StandardCharsets.UTF_8.name()) - val fileContent = try EnvironmentVariableResolver.replaceEnvironmentVariables(source.mkString) finally source.close() + val fileContent = try EnvironmentVariableResolver.resolveFileContent(source.mkString) finally source.close() val mappingJob = org.json4s.jackson.JsonMethods.parse(fileContent).extract[FhirMappingJob] // check there are no duplicate name on mappingTasks of the job val duplicateMappingTasks = FhirMappingJobFormatter.findDuplicateMappingTaskNames(mappingJob.mappings) diff --git a/tofhir-engine/src/test/scala/io/tofhir/test/FhirServerSourceTest.scala b/tofhir-engine/src/test/scala/io/tofhir/test/FhirServerSourceTest.scala index d85ccdb5..33ec40d0 100644 --- a/tofhir-engine/src/test/scala/io/tofhir/test/FhirServerSourceTest.scala +++ b/tofhir-engine/src/test/scala/io/tofhir/test/FhirServerSourceTest.scala @@ -170,7 +170,7 @@ class FhirServerSourceTest extends AsyncFlatSpec with BeforeAndAfterAll with ToF secondRow.getAs[String]("encounterEnd") shouldEqual "2024-08-02T09:00:00Z" secondRow.getAs[String]("observationLoincCode") shouldEqual "85354-9" secondRow.getAs[String]("observationDate") shouldEqual "2024-08-01T10:15:00Z" - secondRow.getAs[String]("observationResult") shouldEqual "140.0 mmHg" + secondRow.getAs[String]("observationResult") shouldEqual "140 mmHg" secondRow.getAs[String]("conditionSnomedCode") shouldEqual "38341003" secondRow.getAs[String]("conditionDate") shouldEqual "2024-08-01" } diff --git a/tofhir-server/src/main/scala/io/tofhir/server/model/Metadata.scala b/tofhir-server/src/main/scala/io/tofhir/server/model/Metadata.scala index eaec0aa1..d75347d3 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/model/Metadata.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/model/Metadata.scala @@ -1,7 +1,7 @@ package io.tofhir.server.model /** - * Model that represents the metadata of the server. + * Model that represents the metadata of the toFHIR server. * * @param name The name of the server. * @param description A description of the server. @@ -21,7 +21,8 @@ case class Metadata(name: String, definitionsRootUrls: Option[Seq[String]], schemasFhirVersion: String, repositoryNames: RepositoryNames, - archiving: Archiving + archiving: Archiving, + environmentVariables: Map[String, String] ) /** diff --git a/tofhir-server/src/main/scala/io/tofhir/server/repository/job/IJobRepository.scala b/tofhir-server/src/main/scala/io/tofhir/server/repository/job/IJobRepository.scala index a081c1e9..03cf2dca 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/repository/job/IJobRepository.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/repository/job/IJobRepository.scala @@ -1,7 +1,7 @@ package io.tofhir.server.repository.job -import io.tofhir.common.model.ICachedRepository import io.tofhir.engine.model.FhirMappingJob +import io.tofhir.engine.repository.ICachedRepository import io.tofhir.server.repository.project.IProjectList import scala.concurrent.Future diff --git a/tofhir-server/src/main/scala/io/tofhir/server/repository/mappingContext/IMappingContextRepository.scala b/tofhir-server/src/main/scala/io/tofhir/server/repository/mappingContext/IMappingContextRepository.scala index 99c4347e..8ee82213 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/repository/mappingContext/IMappingContextRepository.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/repository/mappingContext/IMappingContextRepository.scala @@ -2,7 +2,7 @@ package io.tofhir.server.repository.mappingContext import akka.stream.scaladsl.Source import akka.util.ByteString -import io.tofhir.common.model.ICachedRepository +import io.tofhir.engine.repository.ICachedRepository import io.tofhir.server.model.csv.CsvHeader import io.tofhir.server.repository.project.IProjectList diff --git a/tofhir-server/src/main/scala/io/tofhir/server/repository/schema/ISchemaRepository.scala b/tofhir-server/src/main/scala/io/tofhir/server/repository/schema/ISchemaRepository.scala index e027501a..66d883fc 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/repository/schema/ISchemaRepository.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/repository/schema/ISchemaRepository.scala @@ -2,8 +2,8 @@ package io.tofhir.server.repository.schema import io.onfhir.api.Resource import io.onfhir.definitions.common.model.SchemaDefinition -import io.tofhir.common.model.ICachedRepository import io.tofhir.engine.mapping.schema.IFhirSchemaLoader +import io.tofhir.engine.repository.ICachedRepository import io.tofhir.server.repository.project.IProjectList import scala.concurrent.Future diff --git a/tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/ITerminologySystemRepository.scala b/tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/ITerminologySystemRepository.scala index 892bb571..37e9d2fe 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/ITerminologySystemRepository.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/ITerminologySystemRepository.scala @@ -1,6 +1,6 @@ package io.tofhir.server.repository.terminology -import io.tofhir.common.model.ICachedRepository +import io.tofhir.engine.repository.ICachedRepository import io.tofhir.server.model.TerminologySystem import scala.concurrent.Future diff --git a/tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/TerminologySystemFolderRepository.scala b/tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/TerminologySystemFolderRepository.scala index 935ac9d2..7c89edb8 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/TerminologySystemFolderRepository.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/TerminologySystemFolderRepository.scala @@ -1,7 +1,7 @@ package io.tofhir.server.repository.terminology -import io.tofhir.common.model.ICachedRepository import io.tofhir.engine.Execution.actorSystem.dispatcher +import io.tofhir.engine.repository.ICachedRepository import io.tofhir.engine.util.FileUtils import io.tofhir.server.common.model.{AlreadyExists, BadRequest, ResourceNotFound} import io.tofhir.server.model.TerminologySystem diff --git a/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala b/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala index 98820be6..fc221d21 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala @@ -11,6 +11,7 @@ import io.tofhir.engine.util.FhirMappingJobFormatter.formats import io.tofhir.engine.util.FileUtils import io.tofhir.engine.util.FileUtils.FileExtensions import io.tofhir.engine.{Execution, ToFhirEngine} +import io.tofhir.engine.env.EnvironmentVariableResolver import io.tofhir.rxnorm.RxNormApiFunctionLibraryFactory import io.tofhir.server.common.model.{BadRequest, ResourceNotFound} import io.tofhir.server.model.{ExecuteJobTask, TestResourceCreationRequest} @@ -60,7 +61,9 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin def runJob(projectId: String, jobId: String, executionId: Option[String], executeJobTask: Option[ExecuteJobTask]): Future[Unit] = { jobRepository.getJob(projectId, jobId) map { case None => throw ResourceNotFound("Mapping job does not exists.", s"A mapping job with id $jobId does not exists in the mapping job repository") - case Some(mappingJob) => + case Some(mj) => + val mappingJob = EnvironmentVariableResolver.resolveFhirMappingJob(mj) + // get the list of mapping task to be executed val mappingTasks = executeJobTask.flatMap(_.mappingTaskNames) match { case Some(names) => names.flatMap(name => mappingJob.mappings.find(p => p.name.contentEquals(name))) @@ -198,7 +201,8 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin def testMappingWithJob(projectId: String, jobId: String, testResourceCreationRequest: TestResourceCreationRequest): Future[Seq[FhirMappingResultsForInput]] = { jobRepository.getJob(projectId, jobId) flatMap { case None => throw ResourceNotFound("Mapping job does not exists.", s"A mapping job with id $jobId does not exists in the mapping job repository") - case Some(mappingJob) => + case Some(mj) => + val mappingJob = EnvironmentVariableResolver.resolveFhirMappingJob(mj) logger.debug(s"Testing the mapping ${testResourceCreationRequest.fhirMappingTask.mappingRef} inside the job $jobId by selecting ${testResourceCreationRequest.resourceFilter.numberOfRows} ${testResourceCreationRequest.resourceFilter.order} records.") diff --git a/tofhir-server/src/main/scala/io/tofhir/server/service/MetadataService.scala b/tofhir-server/src/main/scala/io/tofhir/server/service/MetadataService.scala index 10f2a106..66c4550e 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/service/MetadataService.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/service/MetadataService.scala @@ -10,6 +10,7 @@ import io.tofhir.server.config.RedCapServiceConfig import io.onfhir.definitions.resource.fhir.FhirDefinitionsConfig import io.tofhir.engine.Execution.actorSystem import io.tofhir.engine.Execution.actorSystem.dispatcher +import io.tofhir.engine.env.EnvironmentVariableResolver import io.tofhir.server.endpoint.MetadataEndpoint.SEGMENT_METADATA import io.tofhir.server.model.{Archiving, Metadata, RepositoryNames} @@ -56,7 +57,8 @@ class MetadataService(toFhirEngineConfig: ToFhirEngineConfig, erroneousRecordsFolder = toFhirEngineConfig.erroneousRecordsFolder, archiveFolder = toFhirEngineConfig.archiveFolder, streamArchivingFrequency = toFhirEngineConfig.streamArchivingFrequency - ) + ), + environmentVariables = EnvironmentVariableResolver.getEnvironmentVariables ) }