✨ Feat: Add EnvironmentVariable resolving capability to server functi…

…onalities. toFHIR-Web can retrieve the list of related environment variables and the values through /metadata endpoint.

tofhir-common/src/main/scala/io/tofhir/common/env/EnvironmentVariable.scala → tofhir-engine/src/main/scala/io/tofhir/engine/env/EnvironmentVariable.scala

@@ -1,4 +1,4 @@
-package io.tofhir.common.env
+package io.tofhir.engine.env
 
 object EnvironmentVariable extends Enumeration {
   type EnvironmentVariable = Value

tofhir-engine/src/main/scala/io/tofhir/engine/env/EnvironmentVariableResolver.scala

@@ -0,0 +1,110 @@
+package io.tofhir.engine.env
+
+import io.tofhir.engine.model.{FhirMappingJob, FhirRepositorySinkSettings, FhirSinkSettings, FileSystemSourceSettings, MappingJobSourceSettings}
+
+/**
+ * Provide functions to find and resolve the environment variables used within FhirMappingJob definitions.
+ * An ENV_VAR is given as ${ENV_VAR} within the MappingJob definition.
+ */
+object EnvironmentVariableResolver {
+
+  /**
+   * Reads the environment variables defined in the EnvironmentVariable object.
+   *
+   * @return A Map[String, String] containing the environment variable names and their values.
+   */
+  def getEnvironmentVariables: Map[String, String] = {
+    EnvironmentVariable.values
+      .map(_.toString) // Convert Enumeration values to String (names of the variables)
+      .flatMap { envVar =>
+        sys.env.get(envVar).map(envVar -> _) // Get the environment variable value if it exists
+      }
+      .toMap
+  }
+
+  /**
+   * Resolves the environment variables from a given String which is a file content of a FhirMappingJob definition.
+   *
+   * @param fileContent The file content potentially containing placeholders for environment variables.
+   * @return The file content with all recognized environment variables resolved.
+   */
+  def resolveFileContent(fileContent: String): String = {
+    EnvironmentVariable.values.foldLeft(fileContent) { (updatedContent, envVar) =>
+      val placeholder = "\\$\\{" + envVar.toString + "\\}"
+      sys.env.get(envVar.toString) match {
+        case Some(envValue) =>
+          updatedContent.replaceAll(placeholder, envValue)
+        case None =>
+          updatedContent // Leave the content unchanged if the environment variable is not set
+      }
+    }
+  }
+
+  /**
+   * Resolves all environment variable placeholders in a FhirMappingJob instance.
+   *
+   * @param job FhirMappingJob instance.
+   * @return A new FhirMappingJob with resolved values.
+   */
+  def resolveFhirMappingJob(job: FhirMappingJob): FhirMappingJob = {
+    job.copy(
+      sourceSettings = resolveSourceSettings(job.sourceSettings),
+      sinkSettings = resolveSinkSettings(job.sinkSettings)
+    )
+  }
+
+  /**
+   * Resolves environment variables in MappingJobSourceSettings.
+   *
+   * @param sourceSettings A map of source settings.
+   * @return The map with resolved settings.
+   */
+  private def resolveSourceSettings(sourceSettings: Map[String, MappingJobSourceSettings]): Map[String, MappingJobSourceSettings] = {
+    sourceSettings.map { case (key, value) =>
+      key -> (value match {
+        case fs: FileSystemSourceSettings =>
+          fs.copy(dataFolderPath = resolveEnvironmentVariables(fs.dataFolderPath))
+        case other => other // No changes for other types
+      })
+    }
+  }
+
+
+  /**
+   * Resolves environment variables in FhirRepositorySinkSettings.
+   *
+   * @param sinkSettings FhirSinkSettings instance.
+   * @return FhirSinkSettings with resolved fhirRepoUrl, if applicable.
+   */
+  private def resolveSinkSettings(sinkSettings: FhirSinkSettings): FhirSinkSettings = {
+    sinkSettings match {
+      case fr: FhirRepositorySinkSettings =>
+        fr.copy(fhirRepoUrl = resolveEnvironmentVariables(fr.fhirRepoUrl))
+      case other => other // No changes for other types
+    }
+  }
+
+  /**
+   * Resolves placeholders in the format ${ENV_VAR} with their corresponding values
+   * from the system environment variables, but only if the variables are part
+   * of the EnvironmentVariable enumeration.
+   *
+   * @param value The string potentially containing placeholders.
+   * @return The resolved string with placeholders replaced.
+   */
+  private def resolveEnvironmentVariables(value: String): String = {
+    val pattern = """\$\{([A-Z_]+)\}""".r
+
+    pattern.replaceAllIn(value, m => {
+      val envVar = m.group(1)
+      if (EnvironmentVariable.values.exists(_.toString == envVar)) {
+        sys.env.getOrElse(envVar, {
+          throw new RuntimeException(s"Environment variable $envVar is not set.")
+        })
+      } else {
+        throw new RuntimeException(s"Environment variable $envVar is not recognized in EnvironmentVariable enumeration.")
+      }
+    })
+  }
+
+}

tofhir-common/src/main/scala/io/tofhir/common/model/ICachedRepository.scala → tofhir-engine/src/main/scala/io/tofhir/engine/repository/ICachedRepository.scala

@@ -1,4 +1,4 @@
-package io.tofhir.common.model
+package io.tofhir.engine.repository
 
 /**
  * A cached repository.

tofhir-engine/src/main/scala/io/tofhir/engine/repository/mapping/IFhirMappingRepository.scala

@@ -1,7 +1,7 @@
 package io.tofhir.engine.repository.mapping
 
-import io.tofhir.common.model.ICachedRepository
 import io.tofhir.engine.model.FhirMapping
+import io.tofhir.engine.repository.ICachedRepository
 
 trait IFhirMappingRepository extends ICachedRepository {
   /**

tofhir-engine/src/main/scala/io/tofhir/engine/util/FhirMappingJobFormatter.scala

@@ -1,10 +1,10 @@
 package io.tofhir.engine.util
 
 import io.onfhir.client.model.{BasicAuthenticationSettings, BearerTokenAuthorizationSettings, FixedTokenAuthenticationSettings}
-import io.tofhir.common.env.EnvironmentVariableResolver
-import io.tofhir.engine.model.{FhirMappingJob, FhirMappingTask, FhirRepositorySinkSettings, FhirServerSource, FhirServerSourceSettings, FileSystemSinkSettings, FileSystemSource, FileSystemSourceSettings, KafkaSource, KafkaSourceSettings, LocalFhirTerminologyServiceSettings, SQLSchedulingSettings, SchedulingSettings, SqlSource, SqlSourceSettings}
-import org.json4s.{Formats, MappingException, ShortTypeHints}
+import io.tofhir.engine.env.EnvironmentVariableResolver
+import io.tofhir.engine.model._
 import org.json4s.jackson.Serialization
+import org.json4s.{Formats, MappingException, ShortTypeHints}
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Paths}
@@ -60,7 +60,7 @@ object FhirMappingJobFormatter {
    */
   def readMappingJobFromFile(filePath: String): FhirMappingJob = {
     val source = Source.fromFile(filePath, StandardCharsets.UTF_8.name())
-    val fileContent = try EnvironmentVariableResolver.replaceEnvironmentVariables(source.mkString) finally source.close()
+    val fileContent = try EnvironmentVariableResolver.resolveFileContent(source.mkString) finally source.close()
     val mappingJob = org.json4s.jackson.JsonMethods.parse(fileContent).extract[FhirMappingJob]
     // check there are no duplicate name on mappingTasks of the job
     val duplicateMappingTasks = FhirMappingJobFormatter.findDuplicateMappingTaskNames(mappingJob.mappings)

tofhir-engine/src/test/scala/io/tofhir/test/FhirServerSourceTest.scala

@@ -170,7 +170,7 @@ class FhirServerSourceTest extends AsyncFlatSpec with BeforeAndAfterAll with ToF
       secondRow.getAs[String]("encounterEnd") shouldEqual "2024-08-02T09:00:00Z"
       secondRow.getAs[String]("observationLoincCode") shouldEqual "85354-9"
       secondRow.getAs[String]("observationDate") shouldEqual "2024-08-01T10:15:00Z"
-      secondRow.getAs[String]("observationResult") shouldEqual "140.0 mmHg"
+      secondRow.getAs[String]("observationResult") shouldEqual "140 mmHg"
       secondRow.getAs[String]("conditionSnomedCode") shouldEqual "38341003"
       secondRow.getAs[String]("conditionDate") shouldEqual "2024-08-01"
     }

tofhir-server/src/main/scala/io/tofhir/server/model/Metadata.scala

@@ -1,7 +1,7 @@
 package io.tofhir.server.model
 
 /**
- * Model that represents the metadata of the server.
+ * Model that represents the metadata of the toFHIR server.
  *
  * @param name                   The name of the server.
  * @param description            A description of the server.
@@ -21,7 +21,8 @@ case class Metadata(name: String,
                     definitionsRootUrls: Option[Seq[String]],
                     schemasFhirVersion: String,
                     repositoryNames: RepositoryNames,
-                    archiving: Archiving
+                    archiving: Archiving,
+                    environmentVariables: Map[String, String]
                    )
 
 /**

tofhir-server/src/main/scala/io/tofhir/server/repository/job/IJobRepository.scala

@@ -1,7 +1,7 @@
 package io.tofhir.server.repository.job
 
-import io.tofhir.common.model.ICachedRepository
 import io.tofhir.engine.model.FhirMappingJob
+import io.tofhir.engine.repository.ICachedRepository
 import io.tofhir.server.repository.project.IProjectList
 
 import scala.concurrent.Future

tofhir-server/src/main/scala/io/tofhir/server/repository/mappingContext/IMappingContextRepository.scala

@@ -2,7 +2,7 @@ package io.tofhir.server.repository.mappingContext
 
 import akka.stream.scaladsl.Source
 import akka.util.ByteString
-import io.tofhir.common.model.ICachedRepository
+import io.tofhir.engine.repository.ICachedRepository
 import io.tofhir.server.model.csv.CsvHeader
 import io.tofhir.server.repository.project.IProjectList
 

tofhir-server/src/main/scala/io/tofhir/server/repository/schema/ISchemaRepository.scala

@@ -2,8 +2,8 @@ package io.tofhir.server.repository.schema
 
 import io.onfhir.api.Resource
 import io.onfhir.definitions.common.model.SchemaDefinition
-import io.tofhir.common.model.ICachedRepository
 import io.tofhir.engine.mapping.schema.IFhirSchemaLoader
+import io.tofhir.engine.repository.ICachedRepository
 import io.tofhir.server.repository.project.IProjectList
 
 import scala.concurrent.Future

tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/ITerminologySystemRepository.scala

@@ -1,6 +1,6 @@
 package io.tofhir.server.repository.terminology
 
-import io.tofhir.common.model.ICachedRepository
+import io.tofhir.engine.repository.ICachedRepository
 import io.tofhir.server.model.TerminologySystem
 
 import scala.concurrent.Future

tofhir-server/src/main/scala/io/tofhir/server/repository/terminology/TerminologySystemFolderRepository.scala

@@ -1,7 +1,7 @@
 package io.tofhir.server.repository.terminology
 
-import io.tofhir.common.model.ICachedRepository
 import io.tofhir.engine.Execution.actorSystem.dispatcher
+import io.tofhir.engine.repository.ICachedRepository
 import io.tofhir.engine.util.FileUtils
 import io.tofhir.server.common.model.{AlreadyExists, BadRequest, ResourceNotFound}
 import io.tofhir.server.model.TerminologySystem

tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala

@@ -11,6 +11,7 @@ import io.tofhir.engine.util.FhirMappingJobFormatter.formats
 import io.tofhir.engine.util.FileUtils
 import io.tofhir.engine.util.FileUtils.FileExtensions
 import io.tofhir.engine.{Execution, ToFhirEngine}
+import io.tofhir.engine.env.EnvironmentVariableResolver
 import io.tofhir.rxnorm.RxNormApiFunctionLibraryFactory
 import io.tofhir.server.common.model.{BadRequest, ResourceNotFound}
 import io.tofhir.server.model.{ExecuteJobTask, TestResourceCreationRequest}
@@ -60,7 +61,9 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin
   def runJob(projectId: String, jobId: String, executionId: Option[String], executeJobTask: Option[ExecuteJobTask]): Future[Unit] = {
     jobRepository.getJob(projectId, jobId) map {
       case None => throw ResourceNotFound("Mapping job does not exists.", s"A mapping job with id $jobId does not exists in the mapping job repository")
-      case Some(mappingJob) =>
+      case Some(mj) =>
+        val mappingJob = EnvironmentVariableResolver.resolveFhirMappingJob(mj)
+
         // get the list of mapping task to be executed
         val mappingTasks = executeJobTask.flatMap(_.mappingTaskNames) match {
           case Some(names) => names.flatMap(name => mappingJob.mappings.find(p => p.name.contentEquals(name)))
@@ -198,7 +201,8 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin
   def testMappingWithJob(projectId: String, jobId: String, testResourceCreationRequest: TestResourceCreationRequest): Future[Seq[FhirMappingResultsForInput]] = {
     jobRepository.getJob(projectId, jobId) flatMap {
       case None => throw ResourceNotFound("Mapping job does not exists.", s"A mapping job with id $jobId does not exists in the mapping job repository")
-      case Some(mappingJob) =>
+      case Some(mj) =>
+        val mappingJob = EnvironmentVariableResolver.resolveFhirMappingJob(mj)
 
         logger.debug(s"Testing the mapping ${testResourceCreationRequest.fhirMappingTask.mappingRef} inside the job $jobId by selecting ${testResourceCreationRequest.resourceFilter.numberOfRows} ${testResourceCreationRequest.resourceFilter.order} records.")
 

tofhir-server/src/main/scala/io/tofhir/server/service/MetadataService.scala

@@ -10,6 +10,7 @@ import io.tofhir.server.config.RedCapServiceConfig
 import io.onfhir.definitions.resource.fhir.FhirDefinitionsConfig
 import io.tofhir.engine.Execution.actorSystem
 import io.tofhir.engine.Execution.actorSystem.dispatcher
+import io.tofhir.engine.env.EnvironmentVariableResolver
 import io.tofhir.server.endpoint.MetadataEndpoint.SEGMENT_METADATA
 import io.tofhir.server.model.{Archiving, Metadata, RepositoryNames}
 
@@ -56,7 +57,8 @@ class MetadataService(toFhirEngineConfig: ToFhirEngineConfig,
         erroneousRecordsFolder = toFhirEngineConfig.erroneousRecordsFolder,
         archiveFolder = toFhirEngineConfig.archiveFolder,
         streamArchivingFrequency = toFhirEngineConfig.streamArchivingFrequency
-      )
+      ),
+      environmentVariables = EnvironmentVariableResolver.getEnvironmentVariables
     )
   }