refactor: moved intermediate data classes and updated logging

src/main/kotlin/Orchestrator.kt

@@ -9,9 +9,9 @@ import kotlinx.coroutines.withTimeout
 import runner.Runner
 import runner.impl.NodeRunner
 import runner.jvm.JVMRunner
-import technology.idlab.parser.intermediate.IRPipeline
-import technology.idlab.parser.intermediate.IRProcessor
-import technology.idlab.parser.intermediate.IRStage
+import technology.idlab.intermediate.IRPipeline
+import technology.idlab.intermediate.IRProcessor
+import technology.idlab.intermediate.IRStage
 import technology.idlab.util.Log
 
 class Orchestrator(private val pipeline: IRPipeline, processors: List<IRProcessor>) {

src/main/kotlin/extensions/Model.kt

@@ -8,23 +8,31 @@ import org.apache.jena.rdf.model.Resource
 import org.apache.jena.shacl.ShaclValidator
 import technology.idlab.util.Log
 
-/** Validates a model against the SHACL schema defined inside the model itself. */
-internal fun Model.validate(): Model {
+/**
+ * Given an Apache Jena model, run the SHACL validation engine against itself. This means that all
+ * shapes embedded in the model will be used to validate the model itself. If the validation fails,
+ * the program will exit with a fatal error.
+ */
+internal fun Model.validate() {
+  // SHACL runs against the graph, so we need to convert first. Then, simply call a new validation
+  // instance and test the graph against itself.
   val graph = this.graph
   val report = ShaclValidator.get().validate(graph, graph)
 
-  // Exit if the validation failed.
+  // Exit if the validation failed by logging the report.
   if (!report.conforms()) {
     val out = ByteArrayOutputStream()
     report.model.write(out, "TURTLE")
     Log.shared.fatal("Validation failed\n$out")
   }
-
-  return this
 }
 
 /**
  * Return the first object which corresponds to a subject and predicate. Returns null if not found.
+ *
+ * @param resource The subject of the query.
+ * @param property The predicate of the query.
+ * @return The first result of the query, or null if not found.
  */
 internal fun Model.objectOfProperty(resource: Resource, property: Property): RDFNode? {
   return try {
@@ -36,6 +44,10 @@ internal fun Model.objectOfProperty(resource: Resource, property: Property): RDF
 
 /**
  * Return the first subject which corresponds to a predicate and object. Returns null if not found.
+ *
+ * @param property The predicate of the query.
+ * @param obj The object of the query.
+ * @return The first result of the query, or null if not found.
  */
 internal fun Model.subjectWithProperty(property: Property, obj: RDFNode): Resource? {
   return try {

src/main/kotlin/parser/intermediate/IRArgument.kt → src/main/kotlin/intermediate/IRArgument.kt

@@ -1,4 +1,4 @@
-package technology.idlab.parser.intermediate
+package technology.idlab.intermediate
 
 import technology.idlab.util.Log
 

src/main/kotlin/parser/intermediate/IRDependency.kt → src/main/kotlin/intermediate/IRDependency.kt

@@ -1,4 +1,4 @@
-package technology.idlab.parser.intermediate
+package technology.idlab.intermediate
 
 /** A dependency as listed in the configuration file. */
 data class IRDependency(

src/main/kotlin/parser/intermediate/IRPackage.kt → src/main/kotlin/intermediate/IRPackage.kt

@@ -1,4 +1,4 @@
-package technology.idlab.parser.intermediate
+package technology.idlab.intermediate
 
 /** A resolved dependency, with all of its configuration parameters. */
 data class IRPackage(

src/main/kotlin/parser/intermediate/IRParameter.kt → src/main/kotlin/intermediate/IRParameter.kt

@@ -1,4 +1,4 @@
-package technology.idlab.parser.intermediate
+package technology.idlab.intermediate
 
 import technology.idlab.util.Log
 

src/main/kotlin/parser/intermediate/IRPipeline.kt → src/main/kotlin/intermediate/IRPipeline.kt

@@ -1,4 +1,4 @@
-package technology.idlab.parser.intermediate
+package technology.idlab.intermediate
 
 data class IRPipeline(
     val uri: String,

src/main/kotlin/parser/intermediate/IRProcessor.kt → src/main/kotlin/intermediate/IRProcessor.kt

@@ -1,4 +1,4 @@
-package technology.idlab.parser.intermediate
+package technology.idlab.intermediate
 
 import runner.Runner
 

src/main/kotlin/parser/intermediate/IRStage.kt → src/main/kotlin/intermediate/IRStage.kt

@@ -1,4 +1,4 @@
-package technology.idlab.parser.intermediate
+package technology.idlab.intermediate
 
 import arrow.core.zip
 

src/main/kotlin/parser/Parser.kt

@@ -1,16 +1,16 @@
 package technology.idlab.parser
 
 import java.io.File
+import technology.idlab.intermediate.IRPackage
+import technology.idlab.intermediate.IRPipeline
+import technology.idlab.intermediate.IRProcessor
 import technology.idlab.parser.impl.JenaParser
-import technology.idlab.parser.intermediate.IRPackage
-import technology.idlab.parser.intermediate.IRPipeline
-import technology.idlab.parser.intermediate.IRProcessor
 
 /**
  * Parse an RDF file into an intermediate representation, and validate it against the ontology and
  * SHACL shapes.
  */
-abstract class Parser(file: File) {
+abstract class Parser {
   /** The pipelines in the current configuration. */
   abstract val pipelines: List<IRPipeline>
 

src/main/kotlin/parser/impl/JenaParser.kt

@@ -6,25 +6,51 @@ import org.apache.jena.rdf.model.Model
 import org.apache.jena.rdf.model.ModelFactory
 import org.apache.jena.rdf.model.RDFNode
 import org.apache.jena.rdf.model.Resource
+import org.apache.jena.rdf.model.ResourceFactory.createProperty
+import org.apache.jena.rdf.model.ResourceFactory.createResource
 import org.apache.jena.shacl.vocabulary.SHACLM
 import org.apache.jena.vocabulary.RDF
 import runner.Runner
 import technology.idlab.extensions.objectOfProperty
 import technology.idlab.extensions.subjectWithProperty
 import technology.idlab.extensions.validate
+import technology.idlab.intermediate.IRArgument
+import technology.idlab.intermediate.IRDependency
+import technology.idlab.intermediate.IRPackage
+import technology.idlab.intermediate.IRParameter
+import technology.idlab.intermediate.IRPipeline
+import technology.idlab.intermediate.IRProcessor
+import technology.idlab.intermediate.IRStage
 import technology.idlab.parser.Parser
-import technology.idlab.parser.RDFC
-import technology.idlab.parser.intermediate.IRArgument
-import technology.idlab.parser.intermediate.IRDependency
-import technology.idlab.parser.intermediate.IRPackage
-import technology.idlab.parser.intermediate.IRParameter
-import technology.idlab.parser.intermediate.IRPipeline
-import technology.idlab.parser.intermediate.IRProcessor
-import technology.idlab.parser.intermediate.IRStage
 import technology.idlab.resolver.Resolver
 import technology.idlab.util.Log
 
-internal fun Resource.toRunnerTarget(): Runner.Target {
+private class RDFC {
+  companion object {
+    private const val NS = "https://www.rdf-connect.com/#"
+    val NAMESPACE = createResource(NS)!!
+    val processor = createProperty("${NS}Processor")!!
+    val `package` = createProperty("${NS}Package")!!
+    val stage = createProperty("${NS}stage")!!
+    val channel = createProperty("${NS}Channel")!!
+    val target = createProperty("${NS}target")!!
+    val metadata = createProperty("${NS}metadata")!!
+    val arguments = createProperty("${NS}arguments")!!
+    val kotlinRunner = createResource("${NS}Kotlin")!!
+    val dependency = createProperty("${NS}dependency")!!
+    val version = createProperty("${NS}version")!!
+    val author = createProperty("${NS}author")!!
+    val description = createProperty("${NS}description")!!
+    val repo = createProperty("${NS}repo")!!
+    val license = createProperty("${NS}license")!!
+    val prepare = createProperty("${NS}prepare")!!
+    val processors = createProperty("${NS}processors")!!
+    val pipeline = createProperty("${NS}Pipeline")!!
+    val stages = createProperty("${NS}stages")!!
+  }
+}
+
+private fun Resource.toRunnerTarget(): Runner.Target {
   return when (this) {
     RDFC.kotlinRunner -> Runner.Target.JVM
     else -> Log.shared.fatal("Unknown runner type: $this")
@@ -35,7 +61,7 @@ internal fun Resource.toRunnerTarget(): Runner.Target {
  * Maps a resource to an IRParameter.Type based on the URI. Note that this implementation is
  * actually quite slow, and we should probably use Apache Jena native APIs here.
  */
-internal fun Resource.toIRParameterType(): IRParameter.Type {
+private fun Resource.toIRParameterType(): IRParameter.Type {
   return when (this.uri) {
     "http://www.w3.org/2001/XMLSchema#boolean" -> IRParameter.Type.BOOLEAN
     "http://www.w3.org/2001/XMLSchema#byte" -> IRParameter.Type.BYTE
@@ -55,7 +81,7 @@ internal fun Resource.toIRParameterType(): IRParameter.Type {
  * Create a mapping of String to IRParameter from a SHACL property. This is a recursive
  * implementation that will automatically parse nested classes.
  */
-internal fun Model.parseSHACLProperty(property: Resource): Pair<String, IRParameter> {
+private fun Model.parseSHACLProperty(property: Resource): Pair<String, IRParameter> {
   // Retrieve required fields.
   val minCount = objectOfProperty(property, SHACLM.minCount)?.asLiteral()?.int
   val maxCount = objectOfProperty(property, SHACLM.maxCount)?.asLiteral()?.int
@@ -104,7 +130,7 @@ internal fun Model.parseSHACLProperty(property: Resource): Pair<String, IRParame
  * Parse a SHACL shape into a mapping of String to IRParameter. This is a recursive implementation
  * that will automatically parse nested classes.
  */
-internal fun Model.parseSHACLShape(shape: Resource): Map<String, IRParameter> {
+private fun Model.parseSHACLShape(shape: Resource): Map<String, IRParameter> {
   val result = mutableMapOf<String, IRParameter>()
 
   for (property in listObjectsOfProperty(shape, SHACLM.property)) {
@@ -115,7 +141,7 @@ internal fun Model.parseSHACLShape(shape: Resource): Map<String, IRParameter> {
   return result
 }
 
-internal fun Model.nameOfSHACLPath(path: Resource): String {
+private fun Model.nameOfSHACLPath(path: Resource): String {
   val property =
       subjectWithProperty(SHACLM.path, path)
           ?: Log.shared.fatal("No property found for path: $path")
@@ -127,7 +153,7 @@ internal fun Model.nameOfSHACLPath(path: Resource): String {
  * Parse the arguments of a stage. This is a recursive implementation that will automatically parse
  * nested classes. Recursion will continue until all objects found are literals.
  */
-internal fun Model.parseArguments(node: Resource): Map<String, IRArgument> {
+private fun Model.parseArguments(node: Resource): Map<String, IRArgument> {
   val simple = mutableMapOf<String, MutableList<String>>()
   val complex = mutableMapOf<String, MutableList<Map<String, IRArgument>>>()
 
@@ -154,7 +180,7 @@ internal fun Model.parseArguments(node: Resource): Map<String, IRArgument> {
       complex.mapValues { (_, value) -> IRArgument(complex = value) }
 }
 
-internal fun Model.parseProcessor(processor: Resource): IRProcessor {
+private fun Model.parseProcessor(processor: Resource): IRProcessor {
   val uri = processor.toString()
 
   // Determine the target runner.
@@ -191,21 +217,21 @@ internal fun Model.parseProcessor(processor: Resource): IRProcessor {
   return IRProcessor(uri, target, parameters, metadata)
 }
 
-internal fun Model.parseStages(pipeline: Resource): List<IRStage> {
+private fun Model.parseStages(pipeline: Resource): List<IRStage> {
   return listObjectsOfProperty(pipeline, RDFC.stages).toList().map { stage ->
     val processor = objectOfProperty(stage.asResource(), RDF.type)!!.asResource()
     val arguments = objectOfProperty(stage.asResource(), RDFC.arguments)!!.asResource()
     IRStage(stage.toString(), processor.uri, parseArguments(arguments))
   }
 }
 
-internal fun Model.parseDependencies(pipeline: Resource?): List<IRDependency> {
+private fun Model.parseDependencies(pipeline: Resource?): List<IRDependency> {
   return listObjectsOfProperty(pipeline, RDFC.dependency).toList().map { dependency ->
     IRDependency(uri = dependency.toString())
   }
 }
 
-internal fun Model.parsePackage(pkg: Resource): IRPackage {
+private fun Model.parsePackage(pkg: Resource): IRPackage {
   // Get all of its properties.
   val version = objectOfProperty(pkg, RDFC.version)
   val author = objectOfProperty(pkg, RDFC.author)
@@ -228,23 +254,23 @@ internal fun Model.parsePackage(pkg: Resource): IRPackage {
   )
 }
 
-internal fun Model.parsePackages(): List<IRPackage> {
+private fun Model.parsePackages(): List<IRPackage> {
   return listSubjectsWithProperty(RDF.type, RDFC.`package`).toList().map { parsePackage(it) }
 }
 
-internal fun Model.parsePipeline(pipeline: Resource): IRPipeline {
+private fun Model.parsePipeline(pipeline: Resource): IRPipeline {
   return IRPipeline(
       uri = pipeline.uri,
       stages = parseStages(pipeline),
       dependencies = parseDependencies(pipeline),
   )
 }
 
-internal fun Model.parsePipelines(): List<IRPipeline> {
+private fun Model.parsePipelines(): List<IRPipeline> {
   return listSubjectsWithProperty(RDF.type, RDFC.pipeline).toList().map { parsePipeline(it) }
 }
 
-class JenaParser(file: File) : Parser(file) {
+class JenaParser(file: File) : Parser() {
   /** The Apache Jena model. */
   private val model: Model = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM)
 

src/main/kotlin/resolver/Resolver.kt

@@ -2,7 +2,7 @@ package technology.idlab.resolver
 
 import java.io.File
 import org.jetbrains.kotlin.incremental.createDirectory
-import technology.idlab.parser.intermediate.IRDependency
+import technology.idlab.intermediate.IRDependency
 import technology.idlab.resolver.impl.GitResolver
 import technology.idlab.resolver.impl.LocalResolver
 import technology.idlab.util.Log

src/main/kotlin/resolver/impl/GitResolver.kt

@@ -1,7 +1,7 @@
 package technology.idlab.resolver.impl
 
 import java.io.File
-import technology.idlab.parser.intermediate.IRDependency
+import technology.idlab.intermediate.IRDependency
 import technology.idlab.resolver.Resolver
 
 /** Resolve a Git repository by cloning it locally and reading its configuration file. */

src/main/kotlin/resolver/impl/LocalResolver.kt

@@ -1,7 +1,7 @@
 package technology.idlab.resolver.impl
 
 import java.io.File
-import technology.idlab.parser.intermediate.IRDependency
+import technology.idlab.intermediate.IRDependency
 import technology.idlab.resolver.Resolver
 
 /** Resolve a file on the local file system. */

src/main/kotlin/runner/Runner.kt

@@ -1,8 +1,8 @@
 package runner
 
 import kotlinx.coroutines.channels.Channel
-import technology.idlab.parser.intermediate.IRProcessor
-import technology.idlab.parser.intermediate.IRStage
+import technology.idlab.intermediate.IRProcessor
+import technology.idlab.intermediate.IRStage
 import technology.idlab.util.Log
 
 abstract class Runner(

src/main/kotlin/runner/impl/GRPCRunner.kt

@@ -12,10 +12,10 @@ import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.receiveAsFlow
 import runner.Runner
-import technology.idlab.parser.intermediate.IRArgument
-import technology.idlab.parser.intermediate.IRParameter
-import technology.idlab.parser.intermediate.IRProcessor
-import technology.idlab.parser.intermediate.IRStage
+import technology.idlab.intermediate.IRArgument
+import technology.idlab.intermediate.IRParameter
+import technology.idlab.intermediate.IRProcessor
+import technology.idlab.intermediate.IRStage
 import technology.idlab.util.Log
 import technology.idlab.util.retries