diff --git a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Attribute.scala b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Attribute.scala index f6fcc10fb..7791fd354 100644 --- a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Attribute.scala +++ b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Attribute.scala @@ -22,6 +22,4 @@ case class Attribute( dataType: Option[Any], extra: Map[String, Any], name: String, -) extends Vertex { - def this() = this(null, null, null, null, null) -} +) extends Vertex diff --git a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Schema.scala b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Schema.scala index 7db56b5f5..d172295e8 100644 --- a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Schema.scala +++ b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Schema.scala @@ -19,6 +19,4 @@ package za.co.absa.spline.persistence.model case class Schema( override val _key: ArangoDocument.Key, override val _belongsTo: Option[ArangoDocument.Id] -) extends Vertex { - def this() = this(null, null) -} +) extends Vertex diff --git a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/entities.scala b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/entities.scala index 63eda4e12..7fc662221 100644 --- a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/entities.scala +++ b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/entities.scala @@ -49,9 +49,7 @@ case class Edge( index: Option[Edge.Index], path: Option[Edge.FromPath], override val _key: ArangoDocument.Key = null // NOSONAR -) extends ArangoDocument { - def this() = this(null, null, null, null, null, null) // NOSONAR -} +) extends ArangoDocument object Edge { type Index = Int // 0-based number reflecting the position among sibling edges of the same type sharing the same {{_from}} @@ -61,9 +59,7 @@ object Edge { case class DBVersion( version: String, status: String -) extends ArangoDocument with RootEntity { - def this() = this(null, null) // NOSONAR -} +) extends ArangoDocument with RootEntity object DBVersion { def apply(version: String, status: Status.Type): DBVersion = DBVersion(version, status.toString) @@ -79,9 +75,7 @@ object DBVersion { } -case class Counter(override val _key: String, curVal: Long) extends ArangoDocument with RootEntity { - def this() = this(null, Long.MinValue) // NOSONAR -} +case class Counter(override val _key: String, curVal: Long) extends ArangoDocument with RootEntity /** * Represents a named location WHERE data can be read from or written to. @@ -95,7 +89,6 @@ case class DataSource( ) extends Vertex with RootEntity { override val _key: DataSource.Key = null // NOSONAR - def this() = this(null, null, null) // NOSONAR } object DataSource { @@ -150,7 +143,7 @@ case class Progress( ) extends Vertex with RootEntity object Progress { - type JobDurationInNanos = Long + type JobDurationInNanos = java.lang.Long } /** @@ -165,6 +158,4 @@ case class ExecPlanDetails( dataSourceType: String, labels: Map[String, Seq[String]], append: Boolean -) { - def this() = this(null, null, null, null, null, null, null, false) // NOSONAR -} +) diff --git a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/expressions.scala b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/expressions.scala index 9695321b8..936275fd7 100644 --- a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/expressions.scala +++ b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/expressions.scala @@ -31,8 +31,6 @@ case class FunctionalExpression( arity: Int, params: Map[String, Any], ) extends Expression { - def this() = this(null, null, null, null, null, -1, null) - val `type`: String = "Func" } @@ -43,7 +41,5 @@ case class LiteralExpression( override val extra: Map[String, Any], value: Any, ) extends Expression { - def this() = this(null, null, null, null, null) - val `type`: String = "Lit" } diff --git a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/operations.scala b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/operations.scala index dca3e87a6..6bae92761 100644 --- a/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/operations.scala +++ b/arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/operations.scala @@ -44,8 +44,6 @@ case class Read( override val _key: ArangoDocument.Key, override val _belongsTo: Option[ArangoDocument.Id] ) extends Operation { - def this() = this(null, null, null, null, null, null) - override val `type`: Operation.Type = OperationTypes.Read } @@ -58,8 +56,6 @@ case class Write( override val _key: ArangoDocument.Key, override val _belongsTo: Option[ArangoDocument.Id] ) extends Operation { - def this() = this(null, false, null, null, null, null, null) - override val `type`: Operation.Type = OperationTypes.Write } @@ -70,7 +66,5 @@ case class Transformation( override val _key: ArangoDocument.Key, override val _belongsTo: Option[ArangoDocument.Id] ) extends Operation { - def this() = this(null, null, null, null, null) - override val `type`: Operation.Type = OperationTypes.Transformation } diff --git a/arangodb-foxx-services/src/main/services/commons.ts b/arangodb-foxx-services/src/main/services/commons.ts index 4bb04657c..de6c8e776 100644 --- a/arangodb-foxx-services/src/main/services/commons.ts +++ b/arangodb-foxx-services/src/main/services/commons.ts @@ -59,7 +59,7 @@ function getStartDataSourceFromExecutionEvent(startEvent: Progress): DataSource LIMIT 1 RETURN { "_id": ds._key, - "_class": "za.co.absa.spline.consumer.service.model.DataSourceNode", + "_type": "DataSourceNode", "name": ds.uri } `).next() @@ -86,12 +86,12 @@ function getPartialGraphForEvent(event: Progress) { RETURN vertType == "dataSource" ? { "_id": vert._key, - "_class": "za.co.absa.spline.consumer.service.model.DataSourceNode", + "_type": "DataSourceNode", "name": vert.uri } : MERGE(KEEP(vert, ["systemInfo", "agentInfo"]), { "_id": vert._key, - "_class": "za.co.absa.spline.consumer.service.model.ExecutionNode", + "_type": "ExecutionNode", "name": vert.name || "" }) ) diff --git a/build/parent-pom/pom.xml b/build/parent-pom/pom.xml index bf7af9219..ad76833de 100644 --- a/build/parent-pom/pom.xml +++ b/build/parent-pom/pom.xml @@ -341,40 +341,13 @@ - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-modules-base - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-module-paranamer - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-module-jaxb-annotations - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-module-scala_${scala.compat.version} + com.fasterxml.jackson + jackson-bom ${jackson.version} + import + pom + com.twitter finatra-jackson_${scala.compat.version} @@ -527,13 +500,21 @@ ${logback.version} - + com.arangodb arangodb-java-driver 6.25.0 + + com.arangodb + jackson-dataformat-velocypack + 4.3.0 + + + + jakarta.validation jakarta.validation-api diff --git a/consumer-rest-core/src/main/scala/za/co/absa/spline/consumer/rest/controller/OperationDetailsController.scala b/consumer-rest-core/src/main/scala/za/co/absa/spline/consumer/rest/controller/OperationDetailsController.scala index 46b12f881..ef9586278 100644 --- a/consumer-rest-core/src/main/scala/za/co/absa/spline/consumer/rest/controller/OperationDetailsController.scala +++ b/consumer-rest-core/src/main/scala/za/co/absa/spline/consumer/rest/controller/OperationDetailsController.scala @@ -83,7 +83,9 @@ class OperationDetailsController @Autowired() } uses.foldLeft(opProps) { (z, u) => - val relPath = u.path.stripPrefix("$['params']") + val relPath = u.path + .getOrElse(throw new IllegalArgumentException("'path' is missing in the 'uses' type of edge.")) + .stripPrefix("$['params']") val jsonPath = JsonPath.parse(relPath) val nodeId = u.target val node = nodeById(nodeId) diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Attribute.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Attribute.scala index 93adff89b..621dc2f09 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Attribute.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Attribute.scala @@ -19,9 +19,7 @@ package za.co.absa.spline.consumer.service.model case class Attribute( id: String, name: String, - dataTypeId: String) { - def this() = this(null, null, null) -} + dataTypeId: String) object Attribute { type Id = String diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeEdge.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeEdge.scala index 04fa73d2d..2a7cf045e 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeEdge.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeEdge.scala @@ -27,7 +27,5 @@ case class AttributeEdge @ApiModelProperty(value = "Target Node") target: AttributeNode.Id ) extends Graph.Edge { - def this() = this(null, null) - override type JointId = AttributeNode.Id } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeGraph.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeGraph.scala index 667fba773..18ee0a825 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeGraph.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeGraph.scala @@ -27,6 +27,4 @@ case class AttributeGraph( ) extends Graph { override type Node = AttributeNode override type Edge = AttributeEdge - - def this() = this(null, null) } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeNode.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeNode.scala index c7cd83ea7..93a249d1f 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeNode.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeNode.scala @@ -34,8 +34,6 @@ case class AttributeNode transOpIds: Seq[Operation.Id] ) extends Graph.Node { - def this() = this(null, null, null, null) - override type Id = AttributeNode.Id } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/DataSourceInfo.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/DataSourceInfo.scala index 0e1650f80..d7c5cd25e 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/DataSourceInfo.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/DataSourceInfo.scala @@ -24,6 +24,4 @@ case class DataSourceInfo sourceType: String, @ApiModelProperty(value = "Uri of the dataSource") source: String -) { - def this() = this(null, null) -} +) diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/DataSourceNode.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/DataSourceNode.scala deleted file mode 100644 index 0fac57c10..000000000 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/DataSourceNode.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2019 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package za.co.absa.spline.consumer.service.model - -case class DataSourceNode -( - override val _id: String, - override val name: String -) extends LineageOverviewNode { - override type Id = String - - def this() = this(null, null) -} \ No newline at end of file diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/WriteEventInfo.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionEventInfo.scala similarity index 72% rename from consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/WriteEventInfo.scala rename to consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionEventInfo.scala index 1194b4a24..b1ac77f5c 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/WriteEventInfo.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionEventInfo.scala @@ -20,45 +20,46 @@ import za.co.absa.spline.persistence.model.Progress import java.{util => ju} -case class WriteEventInfo +case class ExecutionEventInfo ( - @ApiModelProperty(value = "Id of the execution event") - executionEventId: WriteEventInfo.Id, - @ApiModelProperty(value = "Id of the execution plan") - executionPlanId: ExecutionPlanInfo.Id, - @ApiModelProperty(value = "Name of the framework that triggered this execution event") - frameworkName: String, - @ApiModelProperty(value = "Name of the application/job") - applicationName: String, - @ApiModelProperty(value = "Id of the application/job") - applicationId: String, - @ApiModelProperty(value = "When the execution was triggered") - timestamp: WriteEventInfo.Timestamp, @ApiModelProperty(value = "Duration of execution in nanoseconds (for successful executions)") - durationNs: Option[WriteEventInfo.DurationNs], + durationNs: Option[ExecutionEventInfo.DurationNs], @ApiModelProperty(value = "Error (for failed executions)") - error: Option[WriteEventInfo.Error], + error: Option[Any], @ApiModelProperty(value = "Output data source name") dataSourceName: String, @ApiModelProperty(value = "Output data source URI") dataSourceUri: String, + + // these properties are only applicable for the WRITE execution events, + // therefore they are nullable to avoid deserialization issues of the READ events. + + @ApiModelProperty(value = "Id of the execution event") + executionEventId: ExecutionEventInfo.Id = null, + @ApiModelProperty(value = "Id of the execution plan") + executionPlanId: ExecutionPlanInfo.Id = null, + @ApiModelProperty(value = "Name of the framework that triggered this execution event") + frameworkName: String = null, + @ApiModelProperty(value = "Name of the application/job") + applicationName: String = null, + @ApiModelProperty(value = "Id of the application/job") + applicationId: String = null, + @ApiModelProperty(value = "When the execution was triggered") + timestamp: ExecutionEventInfo.Timestamp = null, @ApiModelProperty(value = "Output data source (or data) type") - dataSourceType: String, + dataSourceType: String = null, @ApiModelProperty(value = "Write mode - (true=Append; false=Override)") - append: WriteEventInfo.Append, + append: ExecutionEventInfo.Append = null, @ApiModelProperty(value = "Other extra info") - extra: Map[String, Any], + extra: Map[String, Any] = null, @ApiModelProperty(value = "Execution event labels") - labels: Option[Map[Label.Name, ju.List[Label.Value]]], -) { - def this() = this(null, null, null, null, null, null, null, null, null, null, null, null, null, null) -} + labels: Map[Label.Name, ju.List[Label.Value]] = null, +) -object WriteEventInfo { +object ExecutionEventInfo { type Id = String type Timestamp = java.lang.Long type DurationNs = Progress.JobDurationInNanos - type Error = Any type Append = java.lang.Boolean } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionPlanInfo.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionPlanInfo.scala index acf11f781..d5fad7441 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionPlanInfo.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionPlanInfo.scala @@ -38,9 +38,7 @@ case class ExecutionPlanInfo inputs: Array[DataSourceInfo], @ApiModelProperty(value = "Write destination") output: DataSourceInfo -) { - def this() = this(null, null, null, null, null, null, null) -} +) object ExecutionPlanInfo { type Id = UUID diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionEdge.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionEdge.scala index df52cc21f..6ad74fb6b 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionEdge.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionEdge.scala @@ -33,13 +33,12 @@ case class ExpressionEdge `type`: ExpressionEdgeType, @ApiModelProperty(value = "JSONPath of inside 'source' entity where this edge is logically attached to (if applicable)") - path: Edge.FromPath, + path: Option[Edge.FromPath], @ApiModelProperty(value = "0-based order (if applicable)") - index: Int -) extends Graph.Edge { - def this() = this(null, null, null, null, -1) + index: Option[Int] +) extends Graph.Edge { override type JointId = ExpressionNode.Id } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionGraph.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionGraph.scala index c36af69cf..14fd0be69 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionGraph.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionGraph.scala @@ -27,8 +27,6 @@ case class ExpressionGraph( ) extends Graph { override type Node = ExpressionNode override type Edge = ExpressionEdge - - def this() = this(null, null) } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionNode.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionNode.scala index 48df72bc5..e97960890 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionNode.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionNode.scala @@ -28,21 +28,18 @@ case class ExpressionNode dataType: Option[String], @ApiModelProperty(value = "Expression name") - name: String, + name: Option[String], @ApiModelProperty(value = "Literal expression value") value: Option[Any], @ApiModelProperty(value = "Expression parameters") - params: Map[String, Any], + params: Map[String, Any] = Map(), @ApiModelProperty(value = "Expression extras") - extra: Map[String, Any], - + extra: Map[String, Any] = Map(), ) extends Graph.Node { - def this() = this(null, null, null, null, null, null) - override type Id = ExpressionNode.Id } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailed.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailed.scala index b3d0ca154..ef5fdf510 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailed.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailed.scala @@ -25,6 +25,4 @@ case class LineageDetailed( executionPlan: ExecutionPlanInfo, @ApiModelProperty(value = "Execution plan level lineage") graph: LineageDetailedGraph -) { - def this() = this(null, null) -} +) diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailedGraph.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailedGraph.scala index 9487f9d95..9bc3382c8 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailedGraph.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailedGraph.scala @@ -25,9 +25,8 @@ case class LineageDetailedGraph nodes: Array[Operation], @ApiModelProperty(value = "List of graph edges showing the triggered operations order") edges: Array[Transition] -) extends Graph { - def this() = this(null, null) +) extends Graph { override type Node = Operation override type Edge = Transition } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverview.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverview.scala index f6ab7bda5..5359df5c4 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverview.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverview.scala @@ -23,6 +23,4 @@ case class LineageOverview( graph: LineageOverviewGraph, @ApiModelProperty(value = "Additional information") info: Map[String, Any] -) { - def this() = this(null, null) -} +) diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverviewGraph.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverviewGraph.scala index 0fd4173f0..35eef9b6f 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverviewGraph.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverviewGraph.scala @@ -28,10 +28,9 @@ case class LineageOverviewGraph @ApiModelProperty(value = "Requested max depth") depthRequested: Int, @ApiModelProperty(value = "Computed depth") - depthComputed: Int, -) extends Graph { - def this() = this(null, null, -1, -1) + depthComputed: Int +) extends Graph { override type Node = LineageOverviewNode override type Edge = Transition } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverviewNode.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverviewNode.scala deleted file mode 100644 index 7f5c8e07f..000000000 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverviewNode.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.spline.consumer.service.model - - -trait LineageOverviewNode extends Graph.Node { - val name: String - type Id = String -} - - - - diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Operation.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Operation.scala index d8998d363..8d7b80975 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Operation.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Operation.scala @@ -31,8 +31,6 @@ case class Operation properties: Map[String, Any] ) extends Graph.Node { override type Id = Operation.Id - - def this() = this(null, null, null, null) } object Operation { diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/OperationDetails.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/OperationDetails.scala index b0fddfa11..53199f9ee 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/OperationDetails.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/OperationDetails.scala @@ -28,10 +28,7 @@ case class OperationDetails @ApiModelProperty(value = "Array of all the schemas") schemas: Array[Array[Attribute]], @ApiModelProperty(value = "Array of indexes of the schemas Array. The schemas at these indexes represent the input schemas") - inputs: Array[Integer], + inputs: Array[Int], @ApiModelProperty(value = "Index of the schemas Array. The schema at this index represents the output schemas") - output: Integer - -) { - def this() = this(null, null, null, null, null) -} + output: Int +) diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageRequest.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageRequest.scala index b4a2fcc5c..b5905af5e 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageRequest.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageRequest.scala @@ -15,6 +15,4 @@ */ package za.co.absa.spline.consumer.service.model -case class PageRequest(page: Int, size: Int) { - def this() = this(0, 0) -} +case class PageRequest(page: Int, size: Int) diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageableExecutionEventsResponse.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageableExecutionEventsResponse.scala index c6e3f2ca6..eceecb1f4 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageableExecutionEventsResponse.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageableExecutionEventsResponse.scala @@ -20,7 +20,7 @@ import io.swagger.annotations.ApiModelProperty case class PageableExecutionEventsResponse( @ApiModelProperty(value = "Array of Execution events") - override val items: Array[WriteEventInfo], + override val items: Array[ExecutionEventInfo], @ApiModelProperty(value = "Total number of executionEvents in the result set") override val totalCount: Long, @ApiModelProperty(value = "Page number") @@ -29,4 +29,4 @@ case class PageableExecutionEventsResponse( override val pageSize: Int, @ApiModelProperty(value = "Total date range (min and max timestamp) of the result set") totalDateRange: Array[Long] -) extends Pageable[WriteEventInfo] +) extends Pageable[ExecutionEventInfo] diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/SortRequest.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/SortRequest.scala index 4d80fc15d..0c8cba34d 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/SortRequest.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/SortRequest.scala @@ -15,6 +15,4 @@ */ package za.co.absa.spline.consumer.service.model -case class SortRequest(sortField: String, sortOrder: String) { - def this() = this(null, null) -} +case class SortRequest(sortField: String, sortOrder: String) diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Transition.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Transition.scala index 806417abe..24a6f045c 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Transition.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Transition.scala @@ -26,7 +26,5 @@ case class Transition @ApiModelProperty(value = "Target Operation") target: Operation.Id ) extends Graph.Edge { - def this() = this(null, null) - override type JointId = Operation.Id } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/dataTypes.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/dataTypes.scala index 592262f74..79e1cce69 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/dataTypes.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/dataTypes.scala @@ -16,6 +16,17 @@ package za.co.absa.spline.consumer.service.model +import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "_typeHint") +@JsonSubTypes(value = Array( + new JsonSubTypes.Type(value = classOf[SimpleDataType], name = "dt.Simple"), + new JsonSubTypes.Type(value = classOf[ArrayDataType], name = "dt.Array"), + new JsonSubTypes.Type(value = classOf[StructDataType], name = "dt.Struct"), +)) sealed trait DataType { def id: String def nullable: Boolean @@ -25,28 +36,21 @@ case class SimpleDataType( override val id: String, override val nullable: Boolean, name: String -) extends DataType { - def this() = this(null, false, null) -} +) extends DataType case class ArrayDataType( override val id: String, override val nullable: Boolean, elementDataTypeId: String -) extends DataType { - def this() = this(null, false, null) -} +) extends DataType case class StructDataType( override val id: String, override val nullable: Boolean, fields: Array[StructField] -) extends DataType { - def this() = this(null, false, null) -} +) extends DataType case class StructField( name: String, - dataTypeId: String) { - def this() = this(null, null) -} + dataTypeId: String +) diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionNode.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/lineageOverviewNodes.scala similarity index 57% rename from consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionNode.scala rename to consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/lineageOverviewNodes.scala index e20051986..06f5d9b52 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExecutionNode.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/lineageOverviewNodes.scala @@ -13,8 +13,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package za.co.absa.spline.consumer.service.model +import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} + + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "_type") +@JsonSubTypes(value = Array( + new JsonSubTypes.Type(value = classOf[DataSourceNode], name = "DataSourceNode"), + new JsonSubTypes.Type(value = classOf[ExecutionNode], name = "ExecutionNode"), +)) +sealed trait LineageOverviewNode extends Graph.Node { + val name: String + type Id = String +} + +case class DataSourceNode +( + override val _id: String, + override val name: String +) extends LineageOverviewNode { + override type Id = String +} + case class ExecutionNode ( override val _id: String, @@ -23,14 +48,10 @@ case class ExecutionNode agentInfo: Option[ExecutionNode.NameAndVersion], ) extends LineageOverviewNode { override type Id = String - - def this() = this(null, null, null, null) } object ExecutionNode { - case class NameAndVersion(name: String, version: String) { - def this() = this(null, null) - } + case class NameAndVersion(name: String, version: String) } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/AbstractExecutionEventRepository.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/AbstractExecutionEventRepository.scala index aa31292eb..6b45de14d 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/AbstractExecutionEventRepository.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/AbstractExecutionEventRepository.scala @@ -15,7 +15,7 @@ */ package za.co.absa.spline.consumer.service.repo -import za.co.absa.spline.consumer.service.model.{Label, PageRequest, SortRequest, WriteEventInfo} +import za.co.absa.spline.consumer.service.model.{Label, PageRequest, SortRequest, ExecutionEventInfo} import scala.concurrent.{ExecutionContext, Future} @@ -41,5 +41,5 @@ trait AbstractExecutionEventRepository { writeAppendOptions: Array[Option[Boolean]], maybeApplicationId: Option[String], maybeDataSourceUri: Option[String]) - (implicit ec: ExecutionContext): Future[(Seq[WriteEventInfo], Long)] + (implicit ec: ExecutionContext): Future[(Seq[ExecutionEventInfo], Long)] } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/DataSourceRepositoryImpl.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/DataSourceRepositoryImpl.scala index 4858c6528..251534747 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/DataSourceRepositoryImpl.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/DataSourceRepositoryImpl.scala @@ -111,11 +111,11 @@ class DataSourceRepositoryImpl @Autowired()(db: ArangoDatabaseAsync) extends Dat writeAppendOptions: Array[Option[Boolean]], maybeWriteApplicationId: Option[String], maybeDataSourceUri: Option[String] - )(implicit ec: ExecutionContext): Future[(Seq[WriteEventInfo], Long)] = { + )(implicit ec: ExecutionContext): Future[(Seq[ExecutionEventInfo], Long)] = { val lblNames = labels.map(_.name) val lblValues = labels.map(_.values) - db.queryAs[WriteEventInfo]( + db.queryAs[ExecutionEventInfo]( s""" |WITH ${SearchViewDef.DataSourceSearchView.name} |FOR ds IN ${SearchViewDef.DataSourceSearchView.name} @@ -159,7 +159,9 @@ class DataSourceRepositoryImpl @Autowired()(db: ArangoDatabaseAsync) extends Dat | "dataSourceType" : ds.lastWriteDetails.execPlanDetails.dataSourceType, | "append" : ds.lastWriteDetails.execPlanDetails.append, | "durationNs" : ds.lastWriteDetails.durationNs, - | "error" : ds.lastWriteDetails.error + | "error" : ds.lastWriteDetails.error, + | "extra" : ds.lastWriteDetails.extra, + | "labels" : ds.lastWriteDetails.labels | } | | SORT resItem.@sortField @sortOrder diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionEventRepositoryImpl.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionEventRepositoryImpl.scala index dab2f0325..1b07beb68 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionEventRepositoryImpl.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionEventRepositoryImpl.scala @@ -106,12 +106,12 @@ class ExecutionEventRepositoryImpl @Autowired()(db: ArangoDatabaseAsync) extends writeAppendOptions: Array[Option[Boolean]], maybeApplicationId: Option[String], maybeDataSourceUri: Option[String] - )(implicit ec: ExecutionContext): Future[(Seq[WriteEventInfo], Long)] = { + )(implicit ec: ExecutionContext): Future[(Seq[ExecutionEventInfo], Long)] = { val lblNames = labels.map(_.name) val lblValues = labels.map(_.values) - db.queryAs[WriteEventInfo]( + db.queryAs[ExecutionEventInfo]( s""" |WITH ${SearchViewDef.ProgressSearchView.name} |FOR ee IN ${SearchViewDef.ProgressSearchView.name} diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionPlanRepositoryImpl.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionPlanRepositoryImpl.scala index 7f143e607..8baa52f14 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionPlanRepositoryImpl.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionPlanRepositoryImpl.scala @@ -83,7 +83,8 @@ class ExecutionPlanRepositoryImpl @Autowired()(db: ArangoDatabaseAsync) extends | "nodes": ops[* RETURN { | "_id" : CURRENT._key, | "_type": CURRENT.type, - | "name" : CURRENT.name || CURRENT.type + | "name" : CURRENT.name || CURRENT.type, + | "properties": {} | }], | "edges": edges[* RETURN { | "source": PARSE_IDENTIFIER(CURRENT._to).key, diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ImpactLineageRepositoryImpl.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ImpactLineageRepositoryImpl.scala index 8d65ae85b..2c08421de 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ImpactLineageRepositoryImpl.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ImpactLineageRepositoryImpl.scala @@ -17,10 +17,11 @@ package za.co.absa.spline.consumer.service.repo import com.arangodb.ArangoDBException import com.arangodb.async.ArangoDatabaseAsync +import com.arangodb.internal.util.ArangoSerializationFactory.Serializer import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Repository import za.co.absa.spline.consumer.service.model.LineageOverview -import za.co.absa.spline.consumer.service.model.WriteEventInfo.Id +import za.co.absa.spline.consumer.service.model.ExecutionEventInfo.Id import java.util.concurrent.CompletionException import scala.PartialFunction.cond @@ -45,7 +46,7 @@ class ImpactLineageRepositoryImpl @Autowired()(db: ArangoDatabaseAsync) extends .route(routeUrl) .get() .asScala - .map(resp => db.util().deserialize[LineageOverview](resp.getBody, classOf[LineageOverview])) + .map(resp => db.util(Serializer.CUSTOM).deserialize[LineageOverview](resp.getBody, classOf[LineageOverview])) .recover({ case ce: CompletionException if cond(ce.getCause)({ case ae: ArangoDBException => ae.getResponseCode == 404 }) => diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ImpactRepository.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ImpactRepository.scala index 6e119f844..34ade9e60 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ImpactRepository.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ImpactRepository.scala @@ -15,12 +15,12 @@ */ package za.co.absa.spline.consumer.service.repo -import za.co.absa.spline.consumer.service.model.{LineageOverview, WriteEventInfo} +import za.co.absa.spline.consumer.service.model.{LineageOverview, ExecutionEventInfo} import scala.concurrent.{ExecutionContext, Future} trait ImpactRepository { - def impactOverviewForExecutionEvent(eventId: WriteEventInfo.Id, maxDepth: Int) + def impactOverviewForExecutionEvent(eventId: ExecutionEventInfo.Id, maxDepth: Int) (implicit ec: ExecutionContext): Future[LineageOverview] } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/LineageRepository.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/LineageRepository.scala index 594ed7afd..235290f53 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/LineageRepository.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/LineageRepository.scala @@ -15,12 +15,12 @@ */ package za.co.absa.spline.consumer.service.repo -import za.co.absa.spline.consumer.service.model.{WriteEventInfo, LineageOverview} +import za.co.absa.spline.consumer.service.model.{ExecutionEventInfo, LineageOverview} import scala.concurrent.{ExecutionContext, Future} trait LineageRepository { - def lineageOverviewForExecutionEvent(eventId: WriteEventInfo.Id, maxDepth: Int) + def lineageOverviewForExecutionEvent(eventId: ExecutionEventInfo.Id, maxDepth: Int) (implicit ec: ExecutionContext): Future[LineageOverview] } diff --git a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/OperationRepositoryImpl.scala b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/OperationRepositoryImpl.scala index c52dcb516..bfd8f4900 100644 --- a/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/OperationRepositoryImpl.scala +++ b/consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/OperationRepositoryImpl.scala @@ -56,18 +56,8 @@ class OperationRepositoryImpl @Autowired()(db: ArangoDatabaseAsync) extends Oper | RETURN schema | ) | - | LET dataTypesFormatted = ( - | LET execPlan = DOCUMENT(ope._belongsTo) - | FOR d IN execPlan.extra.dataTypes || [] - | RETURN MERGE( - | KEEP(d, "id", "name", "fields", "nullable", "elementDataTypeId"), - | { - | "_class": d._typeHint == "dt.Simple" ? "za.co.absa.spline.consumer.service.model.SimpleDataType" - | : d._typeHint == "dt.Array" ? "za.co.absa.spline.consumer.service.model.ArrayDataType" - | : "za.co.absa.spline.consumer.service.model.StructDataType" - | } - | ) - | ) + | LET execPlan = DOCUMENT(ope._belongsTo) + | LET dataTypes = execPlan.extra.dataTypes || [] | | RETURN { | "operation": { @@ -84,7 +74,7 @@ class OperationRepositoryImpl @Autowired()(db: ArangoDatabaseAsync) extends Oper | ope.extra | ) | }, - | "dataTypes": dataTypesFormatted, + | "dataTypes": dataTypes, | "schemas" : schemas, | "inputs" : LENGTH(inputs) > 0 ? RANGE(0, LENGTH(inputs) - 1) : [], | "output" : LENGTH(schemas) - 1 diff --git a/persistence/pom.xml b/persistence/pom.xml index 61ce21475..01754d74b 100644 --- a/persistence/pom.xml +++ b/persistence/pom.xml @@ -87,8 +87,11 @@ com.arangodb - velocypack-module-scala_${scala.compat.version} - 1.2.1 + jackson-dataformat-velocypack + + + com.fasterxml.jackson.module + jackson-module-scala_${scala.compat.version} diff --git a/persistence/src/main/scala/za/co/absa/spline/persistence/ArangoDatabaseFacade.scala b/persistence/src/main/scala/za/co/absa/spline/persistence/ArangoDatabaseFacade.scala index 26201e4d5..f3fe91dc8 100644 --- a/persistence/src/main/scala/za/co/absa/spline/persistence/ArangoDatabaseFacade.scala +++ b/persistence/src/main/scala/za/co/absa/spline/persistence/ArangoDatabaseFacade.scala @@ -17,7 +17,8 @@ package za.co.absa.spline.persistence import com.arangodb.async.{ArangoDBAsync, ArangoDatabaseAsync} -import com.arangodb.velocypack.module.scala.VPackScalaModule +import com.arangodb.mapping.ArangoJack +import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.typesafe.scalalogging.LazyLogging import org.springframework.beans.factory.DisposableBean import za.co.absa.commons.version.Version @@ -36,8 +37,12 @@ class ArangoDatabaseFacade(connectionURL: ArangoConnectionURL, maybeSSLContext: private val isSecure = connectionURL.isSecure private val arango: ArangoDBAsync = { + val arangoJack = new ArangoJack + arangoJack.configure(mapper => mapper + .registerModule(new DefaultScalaModule) + ) val arangoBuilder = new ArangoDBAsync.Builder() - .registerModule(new VPackScalaModule) + .serializer(arangoJack) .having(maybeUser)(_ user _) .having(maybePassword)(_ password _) diff --git a/persistence/src/main/scala/za/co/absa/spline/persistence/tx/FoxxPostTxBuilder.scala b/persistence/src/main/scala/za/co/absa/spline/persistence/tx/FoxxPostTxBuilder.scala index 9a2723950..588fabc47 100644 --- a/persistence/src/main/scala/za/co/absa/spline/persistence/tx/FoxxPostTxBuilder.scala +++ b/persistence/src/main/scala/za/co/absa/spline/persistence/tx/FoxxPostTxBuilder.scala @@ -17,6 +17,7 @@ package za.co.absa.spline.persistence.tx import com.arangodb.async.ArangoDatabaseAsync +import com.arangodb.internal.util.ArangoSerializationFactory.Serializer import scala.concurrent.{ExecutionContext, Future} import scala.jdk.FutureConverters._ @@ -27,9 +28,12 @@ class FoxxPostTxBuilder(endpoint: String, body: AnyRef) extends AbstractTxBuilde override def buildTx(): ArangoTx = new ArangoTx { override def execute[A: ClassTag](db: ArangoDatabaseAsync)(implicit ex: ExecutionContext): Future[A] = { + val serialization = db.util(Serializer.CUSTOM) + val serializedBody = serialization.serialize(body) + db .route(endpoint) - .withBody(body) + .withBody(serializedBody) .post() .asScala .asInstanceOf[Future[A]]