From eb6f28777fb976cdb384b102d3c418dce021e421 Mon Sep 17 00:00:00 2001 From: Robert Isele Date: Thu, 30 Apr 2020 14:15:07 +0200 Subject: [PATCH 1/5] Added applicationData field to Linking Tasks --- .../LinkSpecificationLanguage.xsd | 2 ++ .../serialization/json/JsonSerializers.scala | 31 ++++++++++++------- .../org/silkframework/rule/LinkSpec.scala | 9 ++++-- .../controllers/workspace/TaskApiTest.scala | 5 ++- .../WorkspaceProviderTestTrait.scala | 8 +++-- 5 files changed, 38 insertions(+), 17 deletions(-) diff --git a/silk-core/src/main/resources/org/silkframework/LinkSpecificationLanguage.xsd b/silk-core/src/main/resources/org/silkframework/LinkSpecificationLanguage.xsd index 3cf47cc072..3b5d3906ed 100644 --- a/silk-core/src/main/resources/org/silkframework/LinkSpecificationLanguage.xsd +++ b/silk-core/src/main/resources/org/silkframework/LinkSpecificationLanguage.xsd @@ -285,6 +285,8 @@ + + diff --git a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala index d81bc200e6..9502d2af35 100644 --- a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala +++ b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala @@ -878,6 +878,7 @@ object JsonSerializers { final val REFERENCE_LINKS = "referenceLinks" final val LINK_LIMIT = "linkLimit" final val MATCHING_EXECUTION_TIMEOUT = "matchingExecutionTimeout" + final val APPLICATION_DATA = "applicationData" override def typeNames: Set[String] = Set("Linking") @@ -892,21 +893,29 @@ object JsonSerializers { outputs = mustBeJsArray(mustBeDefined(value, OUTPUTS))(_.value.map(v => Identifier(v.as[JsString].value))), referenceLinks = optionalValue(value, REFERENCE_LINKS).map(fromJson[ReferenceLinks]).getOrElse(ReferenceLinks.empty), linkLimit = numberValueOption(value, LINK_LIMIT).map(_.intValue()).getOrElse(LinkSpec.DEFAULT_LINK_LIMIT), - matchingExecutionTimeout = numberValueOption(value, MATCHING_EXECUTION_TIMEOUT).map(_.intValue()).getOrElse(LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS) + matchingExecutionTimeout = numberValueOption(value, MATCHING_EXECUTION_TIMEOUT).map(_.intValue()).getOrElse(LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS), + applicationData = stringValueOption(value, APPLICATION_DATA) ) } override def write(value: LinkSpec)(implicit writeContext: WriteContext[JsValue]): JsValue = { - Json.obj( - TASKTYPE -> "Linking", - SOURCE -> toJson(value.dataSelections.source), - TARGET -> toJson(value.dataSelections.target), - RULE -> toJson(value.rule), - OUTPUTS -> JsArray(value.outputs.map(id => JsString(id.toString))), - REFERENCE_LINKS -> toJson(value.referenceLinks), - LINK_LIMIT -> JsNumber(value.linkLimit), - MATCHING_EXECUTION_TIMEOUT -> JsNumber(value.matchingExecutionTimeout) - ) + var jsObject = + Json.obj( + TASKTYPE -> "Linking", + SOURCE -> toJson(value.dataSelections.source), + TARGET -> toJson(value.dataSelections.target), + RULE -> toJson(value.rule), + OUTPUTS -> JsArray(value.outputs.map(id => JsString(id.toString))), + REFERENCE_LINKS -> toJson(value.referenceLinks), + LINK_LIMIT -> JsNumber(value.linkLimit), + MATCHING_EXECUTION_TIMEOUT -> JsNumber(value.matchingExecutionTimeout) + ) + + for(data <- value.applicationData) { + jsObject += (APPLICATION_DATA -> JsString(data)) + } + + jsObject } } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala b/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala index 57d8caa682..71e2ea429c 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala @@ -30,7 +30,7 @@ import org.silkframework.runtime.serialization.{ReadContext, ValidatingXMLReader import org.silkframework.runtime.validation.ValidationException import org.silkframework.util._ -import scala.xml.Node +import scala.xml.{Node, NodeSeq} /** * Represents a Silk Link Specification. @@ -40,7 +40,8 @@ case class LinkSpec(dataSelections: DPair[DatasetSelection] = DatasetSelection.e outputs: Seq[Identifier] = Seq.empty, referenceLinks: ReferenceLinks = ReferenceLinks.empty, linkLimit: Int = LinkSpec.DEFAULT_LINK_LIMIT, - matchingExecutionTimeout: Int = LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS) extends TaskSpec { + matchingExecutionTimeout: Int = LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS, + applicationData: Option[String] = None) extends TaskSpec { assert(linkLimit >= 0, "The link limit must be greater equal 0!") assert(matchingExecutionTimeout >= 0, "The matching execution timeout must be greater equal 0!") @@ -190,7 +191,8 @@ object LinkSpec { Identifier(id) }, linkLimit = (node \ "@linkLimit").headOption.map(_.text.toInt).getOrElse(LinkSpec.DEFAULT_LINK_LIMIT), - matchingExecutionTimeout = (node \ "@matchingExecutionTimeout").headOption.map(_.text.toInt).getOrElse(LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS) + matchingExecutionTimeout = (node \ "@matchingExecutionTimeout").headOption.map(_.text.toInt).getOrElse(LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS), + applicationData = (node \ "ApplicationData").headOption.map(_.text) ) } @@ -205,6 +207,7 @@ object LinkSpec { {spec.outputs.map(o => )} + { spec.applicationData.map(data => {data}).getOrElse(NodeSeq.Empty) } } } diff --git a/silk-workbench/silk-workbench-workspace/test/controllers/workspace/TaskApiTest.scala b/silk-workbench/silk-workbench-workspace/test/controllers/workspace/TaskApiTest.scala index 6f212c0f06..82ac01c889 100644 --- a/silk-workbench/silk-workbench-workspace/test/controllers/workspace/TaskApiTest.scala +++ b/silk-workbench/silk-workbench-workspace/test/controllers/workspace/TaskApiTest.scala @@ -216,6 +216,7 @@ class TaskApiTest extends PlaySpec with IntegrationTestTrait { (response.json \ "data" \ "source" \ "typeUri").get mustBe JsString("") (response.json \ "data" \ "target" \ "typeUri").get mustBe JsString("") (response.json \ "data" \ "rule" \ "linkType").get mustBe JsString("owl:sameAs") + (response.json \ "data" \ "applicationData").isEmpty mustBe true } "patch linking task" in { @@ -233,7 +234,8 @@ class TaskApiTest extends PlaySpec with IntegrationTestTrait { | "inputId": "$datasetId", | "typeUri": "", | "restriction": "" - | } + | }, + | "applicationData": "Some Data" | } | } """.stripMargin @@ -251,6 +253,7 @@ class TaskApiTest extends PlaySpec with IntegrationTestTrait { (response.json \ "data" \ "source" \ "typeUri").get mustBe JsString("owl:Class") (response.json \ "data" \ "target" \ "typeUri").get mustBe JsString("") (response.json \ "data" \ "rule" \ "linkType").get mustBe JsString("owl:sameAs") + (response.json \ "data" \ "applicationData").get mustBe JsString("Some Data") } "post workflow task" in { diff --git a/silk-workspace/src/test/scala/org/silkframework/workspace/WorkspaceProviderTestTrait.scala b/silk-workspace/src/test/scala/org/silkframework/workspace/WorkspaceProviderTestTrait.scala index 6283593eda..0af16056df 100644 --- a/silk-workspace/src/test/scala/org/silkframework/workspace/WorkspaceProviderTestTrait.scala +++ b/silk-workspace/src/test/scala/org/silkframework/workspace/WorkspaceProviderTestTrait.scala @@ -83,16 +83,20 @@ trait WorkspaceProviderTestTrait extends FlatSpec with Matchers with MockitoSuga modified = Some(Instant.now) ) + val applicationData = "Some Data" + + val applicationDataUpdated = "Updated Data" + val dataset = PlainTask(DATASET_ID, DatasetSpec(MockDataset("default")), metaData = MetaData(DATASET_ID, Some(DATASET_ID + " description"))) val datasetUpdated = PlainTask(DATASET_ID, DatasetSpec(MockDataset("updated"), uriAttribute = Some("uri")), metaData = MetaData(DATASET_ID)) val linkSpec = LinkSpec(rule = rule, dataSelections = DPair(DatasetSelection(DUMMY_DATASET, ""), DatasetSelection(DUMMY_DATASET, "")), - linkLimit = LinkSpec.DEFAULT_LINK_LIMIT + 1, matchingExecutionTimeout = LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS + 1) + linkLimit = LinkSpec.DEFAULT_LINK_LIMIT + 1, matchingExecutionTimeout = LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS + 1, applicationData = Some(applicationData)) val linkTask = PlainTask(LINKING_TASK_ID, linkSpec, metaData) - val linkTaskUpdated = PlainTask(LINKING_TASK_ID, LinkSpec(rule = rule.copy(operator = None)), metaDataUpdated) + val linkTaskUpdated = PlainTask(LINKING_TASK_ID, LinkSpec(rule = rule.copy(operator = None), applicationData = Some(applicationDataUpdated)), metaDataUpdated) val transformTask = PlainTask( From 93a6f2f205f01042fc5cf3014a6e514296fe7894 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Mon, 9 Nov 2020 12:09:19 +0100 Subject: [PATCH 2/5] Resolve merge conflicts --- .../serialization/json/JsonSerializers.scala | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala index 11bb9959ac..5f68a7e788 100644 --- a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala +++ b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala @@ -939,25 +939,25 @@ object JsonSerializers { } override def write(value: LinkSpec)(implicit writeContext: WriteContext[JsValue]): JsValue = { - var jsObject = Json.obj( - TASKTYPE -> TASK_TYPE_LINKING, - TYPE -> "linking", - PARAMETERS -> Json.obj( - SOURCE -> toJson(value.dataSelections.source), - TARGET -> toJson(value.dataSelections.target), - RULE -> toJson(value.rule), - OUTPUT -> JsString(value.output.map(_.toString).getOrElse("")), - REFERENCE_LINKS -> toJson(value.referenceLinks), - LINK_LIMIT -> JsString(value.linkLimit.toString), - MATCHING_EXECUTION_TIMEOUT -> JsString(value.matchingExecutionTimeout.toString) - ) + var parameters = Json.obj( + SOURCE -> toJson(value.dataSelections.source), + TARGET -> toJson(value.dataSelections.target), + RULE -> toJson(value.rule), + OUTPUT -> JsString(value.output.map(_.toString).getOrElse("")), + REFERENCE_LINKS -> toJson(value.referenceLinks), + LINK_LIMIT -> JsString(value.linkLimit.toString), + MATCHING_EXECUTION_TIMEOUT -> JsString(value.matchingExecutionTimeout.toString) ) for(data <- value.applicationData) { - jsObject += (APPLICATION_DATA -> JsString(data)) + parameters += (APPLICATION_DATA -> JsString(data)) } - jsObject + Json.obj( + TASKTYPE -> TASK_TYPE_LINKING, + TYPE -> "linking", + PARAMETERS -> parameters + ) } } From c961bd0933b8343bab86425975074d9322b2541e Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Mon, 9 Nov 2020 12:16:37 +0100 Subject: [PATCH 3/5] Fix JSON serialization of link spec --- .../org/silkframework/serialization/json/JsonSerializers.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala index 5f68a7e788..a0f7096ecf 100644 --- a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala +++ b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/JsonSerializers.scala @@ -918,7 +918,8 @@ object JsonSerializers { output = stringValueOption(parametersObj, OUTPUT).filter(_.trim.nonEmpty).map(o => Identifier(o.trim)), referenceLinks = optionalValue(parametersObj, REFERENCE_LINKS).map(fromJson[ReferenceLinks]).getOrElse(ReferenceLinks.empty), linkLimit = numberValueOption(parametersObj, LINK_LIMIT).map(_.intValue).getOrElse(LinkSpec.DEFAULT_LINK_LIMIT), - matchingExecutionTimeout = numberValueOption(parametersObj, MATCHING_EXECUTION_TIMEOUT).map(_.intValue).getOrElse(LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS) + matchingExecutionTimeout = numberValueOption(parametersObj, MATCHING_EXECUTION_TIMEOUT).map(_.intValue).getOrElse(LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS), + applicationData = stringValueOption(parametersObj, APPLICATION_DATA) ) } } From 4f4d40339a2fb7ac562c1b19a804d8dd5c60774c Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Mon, 9 Nov 2020 13:53:41 +0100 Subject: [PATCH 4/5] Resolve merge conflicts --- silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala b/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala index a5aa2d6115..2f0003d49c 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala @@ -35,7 +35,7 @@ import org.silkframework.util._ import org.silkframework.workspace.project.task.DatasetTaskReferenceAutoCompletionProvider import scala.collection.mutable -import scala.xml.Node +import scala.xml.{Node, NodeSeq} /** * Represents a Silk Link Specification. From 8e03665ec1d745bbd2135325b37e694380b9fd32 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Mon, 9 Nov 2020 15:03:05 +0100 Subject: [PATCH 5/5] Fix applicationData parameter type --- .../runtime/plugin/ParameterType.scala | 21 ++++++++++++++++++- .../plugin/StringOptionParameter.scala | 11 ++++++++++ .../org/silkframework/rule/LinkSpec.scala | 6 ++++-- 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 silk-core/src/main/scala/org/silkframework/runtime/plugin/StringOptionParameter.scala diff --git a/silk-core/src/main/scala/org/silkframework/runtime/plugin/ParameterType.scala b/silk-core/src/main/scala/org/silkframework/runtime/plugin/ParameterType.scala index c17a3d1d91..beeea913cb 100644 --- a/silk-core/src/main/scala/org/silkframework/runtime/plugin/ParameterType.scala +++ b/silk-core/src/main/scala/org/silkframework/runtime/plugin/ParameterType.scala @@ -182,7 +182,7 @@ object StringParameterType { private val allStaticTypes: Seq[StringParameterType[_]] = { Seq(StringType, CharType, IntType, DoubleType, BooleanType, IntOptionType, StringMapType, UriType, ResourceType, WritableResourceType, ResourceOptionType, ProjectReferenceType, TaskReferenceType, MultilineStringParameterType, SparqlEndpointDatasetParameterType, LongType, - PasswordParameterType, IdentifierType, IdentifierOptionType, StringTraversableParameterType, RestrictionType) + PasswordParameterType, IdentifierType, IdentifierOptionType, StringTraversableParameterType, RestrictionType, StringOptionType) } /** @@ -324,6 +324,25 @@ object StringParameterType { } } + object StringOptionType extends StringParameterType[StringOptionParameter] { + + override def name: String = "option[string]" + + override def description: String = "An optional non-empty string" + + override def fromString(str: String)(implicit prefixes: Prefixes, resourceLoader: ResourceManager): StringOptionParameter = { + if(str.isEmpty) { + StringOptionParameter(None) + } else { + StringOptionParameter(Some(str)) + } + } + + override def toString(value: StringOptionParameter)(implicit prefixes: Prefixes): String = { + value.getOrElse("") + } + } + object IdentifierOptionType extends StringParameterType[IdentifierOptionParameter] { override def name: String = "option[identifier]" diff --git a/silk-core/src/main/scala/org/silkframework/runtime/plugin/StringOptionParameter.scala b/silk-core/src/main/scala/org/silkframework/runtime/plugin/StringOptionParameter.scala new file mode 100644 index 0000000000..591304bf54 --- /dev/null +++ b/silk-core/src/main/scala/org/silkframework/runtime/plugin/StringOptionParameter.scala @@ -0,0 +1,11 @@ +package org.silkframework.runtime.plugin + +/** + * An optional string. Empty strings are interpreted as missing value. + */ +case class StringOptionParameter(value: Option[String]) + +object StringOptionParameter { + implicit def toStringOptionParameter(v: Option[String]): StringOptionParameter = StringOptionParameter(v) + implicit def fromStringOptionParameter(v: StringOptionParameter): Option[String] = v.value +} diff --git a/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala b/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala index 2f0003d49c..b2dc1f6268 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/LinkSpec.scala @@ -25,7 +25,7 @@ import org.silkframework.rule.evaluation.ReferenceLinks import org.silkframework.rule.input.{Input, PathInput, TransformInput} import org.silkframework.rule.similarity.{Aggregation, Comparison, SimilarityOperator} import org.silkframework.runtime.activity.UserContext -import org.silkframework.runtime.plugin.IdentifierOptionParameter +import org.silkframework.runtime.plugin.{IdentifierOptionParameter, StringOptionParameter} import org.silkframework.runtime.plugin.annotations.{Param, Plugin} import org.silkframework.runtime.resource.Resource import org.silkframework.runtime.serialization.XmlSerialization._ @@ -64,7 +64,9 @@ case class LinkSpec(@Param(label = "Source input", value = "The source input to @Param(label = "Matching timeout", value = "The timeout for the matching phase. If the matching takes longer the execution will be stopped.", advanced = true) matchingExecutionTimeout: Int = LinkSpec.DEFAULT_EXECUTION_TIMEOUT_SECONDS, - applicationData: Option[String] = None) extends TaskSpec { + @Param(value = "Additional data that is set by the application managing the linking task.", + visibleInDialog = false) + applicationData: StringOptionParameter = None) extends TaskSpec { assert(linkLimit >= 0, "The link limit must be greater equal 0!") assert(matchingExecutionTimeout >= 0, "The matching execution timeout must be greater equal 0!")