Skip to content

Commit

Permalink
Inclusion of context object for ES Projection (#3245)
Browse files Browse the repository at this point in the history
* feat (composite view): prepare inclusion of context object for ES

* feat (composite view): prepare inclusion of context object for ES

* chore (include context): propagate includeContext param

* feat (include context): conditionally include context object for ES projection

* chore (include context): remove print statement

* feat (incluse context): make method to control inclusion of JSON-LD context

* test (composite view): add test with included context object

* docs (composite view): document the new includeContext flag

* refactor (composite view): reformat code

* refactor (composite view): reformat code

* test (composite view): use separate view id for testing context inclusion

* Updated event serializer tests

* Fixed CompositeViewRoutesSpec

* Added CompositeView integration tests for includeContext=true

* Reordered includeContext field in composite view projection classes

Co-authored-by: Bogdan Roman <[email protected]>
  • Loading branch information
tobiasschweizer and Bogdan Roman authored Jul 23, 2022
1 parent 04e1f6e commit 4e5a887
Show file tree
Hide file tree
Showing 22 changed files with 766 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,8 @@ final class CompositeIndexingStream(
projection.includeMetadata,
projection.includeDeprecated,
sourceAsText = false,
context = projection.context
context = projection.context,
includeContext = projection.includeContext
)
}.runAsyncUnit { bulk =>
// Pushes INDEX/DELETE Elasticsearch bulk operations
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ final case class CompositeIndexingStreamEntry(
includeMetadata: Boolean,
includeDeprecated: Boolean,
sourceAsText: Boolean,
context: ContextValue = ctx
context: ContextValue = ctx,
includeContext: Boolean
): Task[Option[ElasticSearchBulk]] = data match {
case TagNotFound(id) => delete(id, index).map(Some(_))
case resource: IndexingData =>
Expand All @@ -37,7 +38,8 @@ final case class CompositeIndexingStreamEntry(
includeMetadata,
includeDeprecated,
sourceAsText,
context
context,
includeContext
)
else if (containsSchema(resource, resourceSchemas))
delete(resource.id, index).map(Some.apply)
Expand All @@ -57,10 +59,11 @@ final case class CompositeIndexingStreamEntry(
includeMetadata: Boolean,
includeDeprecated: Boolean,
sourceAsText: Boolean,
context: ContextValue
context: ContextValue,
includeContext: Boolean
): Task[Option[ElasticSearchBulk]] = {
if (resource.deprecated && !includeDeprecated) delete(resource.id, idx).map(Some.apply)
else index(resource, idx, includeMetadata, sourceAsText, context)
else index(resource, idx, includeMetadata, sourceAsText, context, includeContext)
}

/**
Expand All @@ -77,9 +80,10 @@ final case class CompositeIndexingStreamEntry(
idx: IndexLabel,
includeMetadata: Boolean,
sourceAsText: Boolean,
context: ContextValue = ctx
context: ContextValue = ctx,
includeContext: Boolean
): Task[Option[ElasticSearchBulk]] =
toDocument(resource, includeMetadata, sourceAsText, context).map { doc =>
toDocument(resource, includeMetadata, sourceAsText, context, includeContext).map { doc =>
Option.when(!doc.isEmpty())(ElasticSearchBulk.Index(idx, resource.id.toString, doc))
}

Expand All @@ -95,29 +99,45 @@ final case class CompositeIndexingStreamEntry(
def containsTypes[A](resource: IndexingData, resourceTypes: Set[Iri]): Boolean =
resourceTypes.isEmpty || resourceTypes.intersect(resource.types).nonEmpty

/**
* Controls whether the JSON-LD context is preserved in the document that is indexed by ES.
*/
private def handleCtxInclusion(json: Json, ctx: ContextValue, includeCtx: Boolean): Json =
if (includeCtx)
json
.removeAllKeys(keywords.context)
.deepMerge(ctx.contextObj.asJson) // remove any existing context before setting the context
else json.removeAllKeys(keywords.context)

private def toDocument(
resource: IndexingData,
includeMetadata: Boolean,
sourceAsText: Boolean,
context: ContextValue
context: ContextValue,
includeContext: Boolean
): Task[Json] = {

val predGraph = resource.graph
val metaGraph = resource.metadataGraph
val graph = if (includeMetadata) predGraph ++ metaGraph else predGraph

if (sourceAsText)
graph
.add(nxv.originalSource.iri, resource.source.noSpaces)
.toCompactedJsonLd(context)
.map(_.obj.asJson)
else if (resource.source.isEmpty())
.map(json => handleCtxInclusion(json, context, includeContext))
else if (resource.source.isEmpty()) {
graph
.toCompactedJsonLd(context)
.map(_.obj.asJson)
else
.map(json => handleCtxInclusion(json, context, includeContext))
} else
(graph -- graph.rootTypesGraph)
.replaceRootNode(BNode.random) // This is done to get rid of the @id in order to avoid overriding the source @id
.toCompactedJsonLd(context)
.map(ld => mergeJsonLd(resource.source, ld.json).removeAllKeys(keywords.context))
.map(ld => mergeJsonLd(resource.source, ld.json))
.map(json => handleCtxInclusion(json, context, includeContext))
}

private def mergeJsonLd(a: Json, b: Json): Json =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ object CompositeViewProjection {
resourceTag: Option[TagLabel],
includeMetadata: Boolean,
includeDeprecated: Boolean,
includeContext: Boolean = false,
permission: Permission,
indexGroup: Option[IndexGroup],
mapping: JsonObject,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ object CompositeViewProjectionFields {
resourceTag: Option[TagLabel] = None,
includeDeprecated: Boolean = false,
includeMetadata: Boolean = false,
includeContext: Boolean = false,
permission: Permission = permissions.query
) extends CompositeViewProjectionFields {
override def tpe: ProjectionType = ElasticSearchProjectionType
Expand All @@ -113,6 +114,7 @@ object CompositeViewProjectionFields {
resourceTag,
includeMetadata,
includeDeprecated,
includeContext,
permission,
indexGroup,
mapping,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ class EventSerializer(system: ExtendedActorSystem) extends SerializerWithStringM
}

implicit final private val configuration: Configuration =
Configuration.default.withDiscriminator(keywords.tpe)
Configuration.default.withDiscriminator(keywords.tpe).withDefaults

implicit private val stringSecretEncryptEncoder: Encoder[Secret[String]] = Encoder.encodeString.contramap {
case Secret(value) => crypto.encrypt(value).get
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
],
"includeDeprecated": false,
"includeMetadata": false,
"includeContext": false,
"permission": "views/query",
"@type": "ElasticSearchProjection"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
"resourceTypes": [
],
"includeDeprecated": false,
"includeContext": false,
"includeMetadata": false,
"permission": "views/query",
"@type": "ElasticSearchProjection"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
],
"includeDeprecated": false,
"includeMetadata": false,
"includeContext": false,
"permission": "views/query",
"@type": "ElasticSearchProjection",
"_uuid": "{{uuid}}"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
{
"@type": "CompositeViewCreated",
"id": "http://example.com/composite-view",
"instant": "1970-01-01T00:00:00Z",
"project": "myorg/myproj",
"rev": 1,
"source": {
"projections": [
{
"@id": "http://example.com/blazegraph-projection",
"@type": "SparqlProjection",
"includeDeprecated": false,
"includeMetadata": false,
"permission": "views/query",
"query": "prefix p: <http://localhost/>\nCONSTRUCT{ {resource_id} p:transformed ?v } WHERE { {resource_id} p:predicate ?v}",
"resourceSchemas": [
],
"resourceTypes": [
]
},
{
"@id": "http://example.com/es-projection",
"@type": "ElasticSearchProjection",
"context": {
},
"includeDeprecated": false,
"includeMetadata": false,
"mapping": {
},
"permission": "views/query",
"query": "prefix p: <http://localhost/>\nCONSTRUCT{ {resource_id} p:transformed ?v } WHERE { {resource_id} p:predicate ?v}",
"resourceSchemas": [
],
"resourceTypes": [
],
"settings": {
}
}
],
"rebuildStrategy": {
"@type": "Interval",
"value": "1 minute"
},
"sources": [
{
"@id": "http://example.com/cross-project-source",
"@type": "CrossProjectEventStream",
"identities": [
{
"@id": "http://localhost/v1/anonymous",
"@type": "Anonymous"
}
],
"includeDeprecated": false,
"project": "org/otherproject",
"resourceSchemas": [
],
"resourceTypes": [
]
},
{
"@id": "http://example.com/remote-project-source",
"@type": "RemoteProjectEventStream",
"endpoint": "http://example.com/remote-endpoint",
"includeDeprecated": false,
"project": "org/remoteproject",
"resourceSchemas": [
],
"resourceTypes": [
]
},
{
"@id": "http://example.com/project-source",
"@type": "ProjectEventStream",
"includeDeprecated": false,
"resourceSchemas": [
],
"resourceTypes": [
]
}
]
},
"subject": {
"@type": "User",
"realm": "myrealm",
"subject": "myuser"
},
"uuid": "{{uuid}}",
"value": {
"projections": [
{
"@type": "SparqlProjection",
"id": "http://example.com/blazegraph-projection",
"includeDeprecated": false,
"includeMetadata": false,
"permission": "views/query",
"query": "prefix p: <http://localhost/>\nCONSTRUCT{ {resource_id} p:transformed ?v } WHERE { {resource_id} p:predicate ?v}",
"resourceSchemas": [
],
"resourceTypes": [
],
"uuid": "{{uuid}}"
},
{
"@type": "ElasticSearchProjection",
"context": {
},
"id": "http://example.com/es-projection",
"includeDeprecated": false,
"includeMetadata": false,
"mapping": {
},
"permission": "views/query",
"query": "prefix p: <http://localhost/>\nCONSTRUCT{ {resource_id} p:transformed ?v } WHERE { {resource_id} p:predicate ?v}",
"resourceSchemas": [
],
"resourceTypes": [
],
"settings": {
},
"uuid": "{{uuid}}"
}
],
"rebuildStrategy": {
"@type": "Interval",
"value": "1 minute"
},
"sources": [
{
"@type": "RemoteProjectSource",
"endpoint": "http://example.com/remote-endpoint",
"id": "http://example.com/remote-project-source",
"includeDeprecated": false,
"project": "org/remoteproject",
"resourceSchemas": [
],
"resourceTypes": [
],
"token": {
"value": "vv/MQBHmWaNm+TX/EigReQ=="
},
"uuid": "{{uuid}}"
},
{
"@type": "ProjectSource",
"id": "http://example.com/project-source",
"includeDeprecated": false,
"resourceSchemas": [
],
"resourceTypes": [
],
"uuid": "{{uuid}}"
},
{
"@type": "CrossProjectSource",
"id": "http://example.com/cross-project-source",
"identities": [
{
"@type": "Anonymous"
}
],
"includeDeprecated": false,
"project": "org/otherproject",
"resourceSchemas": [
],
"resourceTypes": [
],
"uuid": "{{uuid}}"
}
]
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
},
"includeDeprecated": false,
"includeMetadata": false,
"includeContext": false,
"mapping": {
},
"permission": "views/query",
Expand Down Expand Up @@ -108,6 +109,7 @@
"id": "http://example.com/es-projection",
"includeDeprecated": false,
"includeMetadata": false,
"includeContext": false,
"mapping": {
},
"permission": "views/query",
Expand Down
Loading

0 comments on commit 4e5a887

Please sign in to comment.