Skip to content

Commit

Permalink
✨ feat: Implement reload endpoint. (#249)
Browse files Browse the repository at this point in the history
* ✨ feat: Implement reload endpoint.

* ♻️ refactor: Implement mapping clearing in more memory efficient way.

* 🐛 build: Fix import error.
  • Loading branch information
Okanmercan99 authored Nov 19, 2024
1 parent 2621866 commit 364d2a0
Show file tree
Hide file tree
Showing 9 changed files with 220 additions and 28 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package io.tofhir.server.endpoint

import akka.http.scaladsl.model.StatusCodes
import akka.http.scaladsl.server.Directives.{complete, get, pathEndOrSingleSlash, pathPrefix}
import akka.http.scaladsl.server.Route
import com.typesafe.scalalogging.LazyLogging
import io.tofhir.engine.Execution.actorSystem.dispatcher
import io.tofhir.server.common.model.ToFhirRestCall
import io.tofhir.server.endpoint.ReloadEndpoint.SEGMENT_RELOAD
import io.tofhir.server.repository.job.JobFolderRepository
import io.tofhir.server.repository.mapping.ProjectMappingFolderRepository
import io.tofhir.server.repository.mappingContext.MappingContextFolderRepository
import io.tofhir.server.repository.schema.SchemaFolderRepository
import io.tofhir.server.repository.terminology.TerminologySystemFolderRepository
import io.tofhir.server.service.ReloadService
import io.tofhir.server.service.db.FolderDBInitializer

/**
* Endpoint to reload resources from the file system.
* */
class ReloadEndpoint(mappingRepository: ProjectMappingFolderRepository,
schemaRepository: SchemaFolderRepository,
mappingJobRepository: JobFolderRepository,
mappingContextRepository: MappingContextFolderRepository,
terminologySystemFolderRepository: TerminologySystemFolderRepository,
folderDBInitializer: FolderDBInitializer) extends LazyLogging {

val reloadService: ReloadService = new ReloadService(
mappingRepository,
schemaRepository,
mappingJobRepository,
mappingContextRepository,
terminologySystemFolderRepository,
folderDBInitializer
)

def route(request: ToFhirRestCall): Route = {
pathPrefix(SEGMENT_RELOAD) {
pathEndOrSingleSlash {
reloadResources
}
}
}

/**
* Route to reload all resources
* @return
*/
private def reloadResources: Route = {
get {
complete {
reloadService.reloadResources() map { _ =>
StatusCodes.NoContent
}
}
}
}
}

object ReloadEndpoint {
val SEGMENT_RELOAD = "reload"
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ class ToFhirServerEndpoint(toFhirEngineConfig: ToFhirEngineConfig, webServerConf
val codeSystemRepository: ICodeSystemRepository = new CodeSystemRepository(toFhirEngineConfig.terminologySystemFolderPath)

// Initialize the projects by reading the resources available in the file system
new FolderDBInitializer(schemaRepository, mappingRepository, mappingJobRepository, projectRepository, mappingContextRepository).init()
val folderDBInitializer = new FolderDBInitializer(schemaRepository, mappingRepository, mappingJobRepository, projectRepository, mappingContextRepository)
folderDBInitializer.init()

val projectEndpoint = new ProjectEndpoint(schemaRepository, mappingRepository, mappingJobRepository, mappingContextRepository, projectRepository)
val fhirDefinitionsEndpoint = new FhirDefinitionsEndpoint(fhirDefinitionsConfig)
Expand All @@ -49,6 +50,7 @@ class ToFhirServerEndpoint(toFhirEngineConfig: ToFhirEngineConfig, webServerConf
val fileSystemTreeStructureEndpoint = new FileSystemTreeStructureEndpoint()
val terminologyServiceManagerEndpoint = new TerminologyServiceManagerEndpoint(terminologySystemFolderRepository, conceptMapRepository, codeSystemRepository, mappingJobRepository)
val metadataEndpoint = new MetadataEndpoint(toFhirEngineConfig, webServerConfig, fhirDefinitionsConfig, redCapServiceConfig)
val reloadEndpoint= new ReloadEndpoint(mappingRepository, schemaRepository, mappingJobRepository, mappingContextRepository, terminologySystemFolderRepository.asInstanceOf[TerminologySystemFolderRepository], folderDBInitializer)
// Custom rejection handler to send proper messages to user
val toFhirRejectionHandler: RejectionHandler = ToFhirRejectionHandler.getRejectionHandler()

Expand All @@ -69,7 +71,8 @@ class ToFhirServerEndpoint(toFhirEngineConfig: ToFhirEngineConfig, webServerConf
fhirDefinitionsEndpoint.route(),
fhirPathFunctionsEndpoint.route(),
fileSystemTreeStructureEndpoint.route(restCall),
metadataEndpoint.route(restCall)
metadataEndpoint.route(restCall),
reloadEndpoint.route(restCall),
) ++ redcapEndpoint.map(_.route(restCall))

concat(routes: _*)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ class JobFolderRepository(jobRepositoryFolderPath: String, projectFolderReposito

private val logger: Logger = Logger(this.getClass)
// project id -> mapping job id -> mapping job
private val jobDefinitions: mutable.Map[String, mutable.Map[String, FhirMappingJob]] = initMap(jobRepositoryFolderPath)
private val jobDefinitions: mutable.Map[String, mutable.Map[String, FhirMappingJob]] = mutable.Map.empty[String, mutable.Map[String, FhirMappingJob]]
// Initialize the map for the first time
initMap(jobRepositoryFolderPath)

/**
* Returns the mappings managed by this repository
Expand Down Expand Up @@ -183,8 +185,7 @@ class JobFolderRepository(jobRepositoryFolderPath: String, projectFolderReposito
* @param jobRepositoryFolderPath folder path to the job repository
* @return
*/
private def initMap(jobRepositoryFolderPath: String): mutable.Map[String, mutable.Map[String, FhirMappingJob]] = {
val map = mutable.Map.empty[String, mutable.Map[String, FhirMappingJob]]
private def initMap(jobRepositoryFolderPath: String): Unit = {
val jobRepositoryFolder = FileUtils.getPath(jobRepositoryFolderPath).toFile
logger.info(s"Initializing the Mapping Job Repository from path ${jobRepositoryFolder.getAbsolutePath}.")
if (!jobRepositoryFolder.exists()) {
Expand Down Expand Up @@ -217,8 +218,16 @@ class JobFolderRepository(jobRepositoryFolderPath: String, projectFolderReposito
System.exit(1)
}
}
map.put(projectDirectory.getName, fhirJobMap)
this.jobDefinitions.put(projectDirectory.getName, fhirJobMap)
}
map
}

/**
* Reload the job definitions from the given folder
* @return
*/
def reloadJobDefinitions(): Unit = {
this.jobDefinitions.clear()
initMap(jobRepositoryFolderPath)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ class ProjectMappingFolderRepository(mappingRepositoryFolderPath: String, projec
private val logger: Logger = Logger(this.getClass)

// project id -> mapping id -> mapping
private val mappingDefinitions: mutable.Map[String, mutable.Map[String, FhirMapping]] = initMap(mappingRepositoryFolderPath)
private val mappingDefinitions: mutable.Map[String, mutable.Map[String, FhirMapping]] = mutable.Map.empty[String, mutable.Map[String, FhirMapping]]
// Initialize the map for the first time
initMap(mappingRepositoryFolderPath)

/**
* Returns the mappings managed by this repository
Expand Down Expand Up @@ -201,8 +203,7 @@ class ProjectMappingFolderRepository(mappingRepositoryFolderPath: String, projec
* @param mappingRepositoryFolderPath path to the mapping repository
* @return
*/
private def initMap(mappingRepositoryFolderPath: String): mutable.Map[String, mutable.Map[String, FhirMapping]] = {
val map = mutable.Map.empty[String, mutable.Map[String, FhirMapping]]
private def initMap(mappingRepositoryFolderPath: String): Unit = {
val mappingRepositoryFolder = FileUtils.getPath(mappingRepositoryFolderPath).toFile
if (!mappingRepositoryFolder.exists()) {
mappingRepositoryFolder.mkdirs()
Expand Down Expand Up @@ -232,10 +233,9 @@ class ProjectMappingFolderRepository(mappingRepositoryFolderPath: String, projec
// No processable schema files under projectDirectory
logger.warn(s"There are no processable mapping files under ${projectDirectory.getAbsolutePath}. Skipping ${projectDirectory.getName}.")
} else {
map.put(projectDirectory.getName, fhirMappingMap)
this.mappingDefinitions.put(projectDirectory.getName, fhirMappingMap)
}
}
map
}

/**
Expand All @@ -257,4 +257,13 @@ class ProjectMappingFolderRepository(mappingRepositoryFolderPath: String, projec
override def invalidate(): Unit = {
// nothing needs to be done as we keep the cache always up-to-date
}

/**
* Reload the mapping definitions from the given folder
* @return
*/
def reloadMappingDefinitions(): Unit = {
this.mappingDefinitions.clear()
initMap(mappingRepositoryFolderPath)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ import scala.concurrent.Future
*/
class MappingContextFolderRepository(mappingContextRepositoryFolderPath: String, projectFolderRepository: ProjectFolderRepository) extends IMappingContextRepository {
// project id -> mapping context id
private val mappingContextDefinitions: mutable.Map[String, Seq[String]] = initMap(mappingContextRepositoryFolderPath)
private val mappingContextDefinitions: mutable.Map[String, Seq[String]] = mutable.Map.empty[String, Seq[String]]
// Initialize the map for the first time
initMap(mappingContextRepositoryFolderPath)

/**
* Returns the mapping context cached in memory
Expand Down Expand Up @@ -230,8 +232,7 @@ class MappingContextFolderRepository(mappingContextRepositoryFolderPath: String,
* @param mappingContextRepositoryFolderPath path to the mapping context repository
* @return
*/
private def initMap(mappingContextRepositoryFolderPath: String): mutable.Map[String, Seq[String]] = {
val map = mutable.Map.empty[String, Seq[String]]
private def initMap(mappingContextRepositoryFolderPath: String): Unit = {
val folder = FileUtils.getPath(mappingContextRepositoryFolderPath).toFile
if (!folder.exists()) {
folder.mkdirs()
Expand All @@ -241,9 +242,16 @@ class MappingContextFolderRepository(mappingContextRepositoryFolderPath: String,
directories.foreach { projectDirectory =>
val files = IOUtil.getFilesFromFolder(projectDirectory, withExtension = None, recursively = Some(true))
val fileNameList = files.map(_.getName)
map.put(projectDirectory.getName, fileNameList)
this.mappingContextDefinitions.put(projectDirectory.getName, fileNameList)
}
map
}

/**
* Reload the mapping context definitions from the given folder
* @return
*/
def reloadMappingContextDefinitions(): Unit = {
this.mappingContextDefinitions.clear()
initMap(mappingContextRepositoryFolderPath)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ class SchemaFolderRepository(schemaRepositoryFolderPath: String, projectFolderRe
private val baseFhirConfig: BaseFhirConfig = initBaseFhirConfig(fhirConfigReader)
private val simpleStructureDefinitionService = new SimpleStructureDefinitionService(baseFhirConfig)
// Schema definition cache: project id -> schema id -> schema definition
private val schemaDefinitions: mutable.Map[String, mutable.Map[String, SchemaDefinition]] = initMap(schemaRepositoryFolderPath)
private val schemaDefinitions: mutable.Map[String, mutable.Map[String, SchemaDefinition]] = mutable.Map.empty[String, mutable.Map[String, SchemaDefinition]]
// Initialize the map for the first time
initMap(schemaRepositoryFolderPath)

/**
* Returns the schema cached schema definitions by this repository
Expand Down Expand Up @@ -235,13 +237,12 @@ class SchemaFolderRepository(schemaRepositoryFolderPath: String, projectFolderRe
}

/**
* Parses the given schema folder and creates a SchemaDefinition map
* Parses the given schema folder and initialize a SchemaDefinition map
*
* @param schemaRepositoryFolderPath
* @return
*/
private def initMap(schemaRepositoryFolderPath: String): mutable.Map[String, mutable.Map[String, SchemaDefinition]] = {
val schemaDefinitionMap = mutable.Map[String, mutable.Map[String, SchemaDefinition]]()
private def initMap(schemaRepositoryFolderPath: String): Unit = {
val schemaFolder = FileUtils.getPath(schemaRepositoryFolderPath).toFile
logger.info(s"Initializing the Schema Repository from path ${schemaFolder.getAbsolutePath}.")
if (!schemaFolder.exists()) {
Expand Down Expand Up @@ -283,10 +284,9 @@ class SchemaFolderRepository(schemaRepositoryFolderPath: String, projectFolderRe
// No processable schema files under projectFolder
logger.warn(s"There are no processable schema files under ${projectFolder.getAbsolutePath}. Skipping ${projectFolder.getName}.")
} else {
schemaDefinitionMap.put(projectFolder.getName, projectSchemas)
this.schemaDefinitions.put(projectFolder.getName, projectSchemas)
}
})
schemaDefinitionMap
}

/**
Expand Down Expand Up @@ -521,5 +521,14 @@ class SchemaFolderRepository(schemaRepositoryFolderPath: String, projectFolderRe
}
}
}

/**
* Reload the schema definitions from the given folder
* @return
*/
def reloadSchemaDefinitions(): Unit = {
this.schemaDefinitions.clear()
initMap(schemaRepositoryFolderPath)
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ import scala.concurrent.Future
class TerminologySystemFolderRepository(terminologySystemsFolderPath: String) extends ITerminologySystemRepository {

// terminology system id -> TerminologySystem
private val terminologySystemMap: mutable.Map[String, TerminologySystem] = initMap()
private val terminologySystemMap: mutable.Map[String, TerminologySystem] = mutable.Map.empty[String, TerminologySystem]
// Initialize the map for the first time
initMap()

/**
* Retrieve the metadata of all TerminologySystems
Expand Down Expand Up @@ -212,13 +214,11 @@ class TerminologySystemFolderRepository(terminologySystemsFolderPath: String) ex
*
* @return
*/
private def initMap(): mutable.Map[String, TerminologySystem] = {
private def initMap(): Unit = {
val termonologySystems = readTerminologySystemsDBFile()
val map = mutable.Map[String, TerminologySystem]()
termonologySystems.foreach(terminology => {
map.put(terminology.id, terminology)
this.terminologySystemMap.put(terminology.id, terminology)
})
map
}

/**
Expand Down Expand Up @@ -252,6 +252,14 @@ class TerminologySystemFolderRepository(terminologySystemsFolderPath: String) ex
FileOperations.writeJsonContent(localTerminologyFile, terminologySystems)
}

/**
* Reload the terminology systems from the given folder
* @return
*/
def reloadTerminologySystems(): Unit = {
this.terminologySystemMap.clear()
initMap()
}
}

object TerminologySystemFolderRepository {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package io.tofhir.server.service

import io.tofhir.server.repository.job.JobFolderRepository
import io.tofhir.server.repository.mapping.ProjectMappingFolderRepository
import io.tofhir.server.repository.mappingContext.MappingContextFolderRepository
import io.tofhir.server.repository.schema.SchemaFolderRepository
import io.tofhir.server.repository.terminology.TerminologySystemFolderRepository
import io.tofhir.engine.Execution.actorSystem.dispatcher
import io.tofhir.server.service.db.FolderDBInitializer

import scala.concurrent.Future

/**
* Service for reloading resources from the file system.
*/
class ReloadService(mappingRepository: ProjectMappingFolderRepository,
schemaRepository: SchemaFolderRepository,
mappingJobRepository: JobFolderRepository,
mappingContextRepository: MappingContextFolderRepository,
terminologySystemFolderRepository: TerminologySystemFolderRepository,
folderDBInitializer: FolderDBInitializer) {

/**
* Reload all resources.
* @return
*/
def reloadResources(): Future[Unit] = {
Future{
mappingRepository.reloadMappingDefinitions()
schemaRepository.reloadSchemaDefinitions()
mappingJobRepository.reloadJobDefinitions()
mappingContextRepository.reloadMappingContextDefinitions()
terminologySystemFolderRepository.reloadTerminologySystems()
folderDBInitializer.init()
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package io.tofhir.server.endpoint

import akka.http.scaladsl.model.StatusCodes
import io.tofhir.engine.util.FileUtils
import io.tofhir.server.BaseEndpointTest
import io.tofhir.server.model.Project
import io.onfhir.definitions.common.model.Json4sSupport.formats
import io.tofhir.server.repository.project.ProjectFolderRepository
import io.tofhir.server.util.FileOperations
import org.json4s.{JArray, JObject, JString}
import org.json4s.jackson.{JsonMethods, Serialization}

import java.io.FileWriter

class ReloadingEndpointTest extends BaseEndpointTest {

"The service" should {

"should reload projects successfully after updating project file" in {
// Read projects and update description fields from projects.json
val projectsFile = FileUtils.getPath(ProjectFolderRepository.PROJECTS_JSON).toFile
val parsedProjects = FileOperations.readFileIntoJson(projectsFile).asInstanceOf[JArray].arr.map(p => p.asInstanceOf[JObject])
val reloadedProjects = parsedProjects.map(project => {
project.mapField {
case ("description", _) => ("description", JString("reloaded"))
case otherwise => otherwise
}
})
val fw = new FileWriter(projectsFile)
try fw.write(Serialization.writePretty(reloadedProjects)) finally fw.close()

// Trigger reload endpoint
Get(s"/${webServerConfig.baseUri}/${ReloadEndpoint.SEGMENT_RELOAD}") ~> route ~> check {
status shouldEqual StatusCodes.NoContent

// Check project endpoint for whether project descriptions are updated
Get(s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}") ~> route ~> check {
status shouldEqual StatusCodes.OK
val projects: Seq[Project] = JsonMethods.parse(responseAs[String]).extract[Seq[Project]]
projects.foreach(project => {
project.description.get shouldEqual "reloaded"
})
}
}
}
}
}

0 comments on commit 364d2a0

Please sign in to comment.