Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add option to query Graph Analytics views #4148

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaSchemeDirectives
import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities
import ch.epfl.bluebrain.nexus.delta.sdk.model._
import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, Projects}
import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext.ContextRejection
import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, Projects}
import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
import ch.epfl.bluebrain.nexus.delta.sourcing.config.QueryConfig
import ch.epfl.bluebrain.nexus.delta.sourcing.projections.Projections
Expand Down Expand Up @@ -50,6 +50,10 @@ class GraphAnalyticsPluginModule(priority: Int) extends ModuleDef {
GraphAnalyticsCoordinator(projects, analyticsStream, supervisor, client, config)
}

make[GraphAnalyticsViewsQuery].from { (client: ElasticSearchClient, config: GraphAnalyticsConfig) =>
new GraphAnalyticsViewsQueryImpl(config.prefix, client)
}

make[GraphAnalyticsRoutes].from {
(
identities: Identities,
Expand All @@ -60,14 +64,16 @@ class GraphAnalyticsPluginModule(priority: Int) extends ModuleDef {
baseUri: BaseUri,
s: Scheduler,
cr: RemoteContextResolution @Id("aggregate"),
ordering: JsonKeyOrdering
ordering: JsonKeyOrdering,
viewsQuery: GraphAnalyticsViewsQuery
) =>
new GraphAnalyticsRoutes(
identities,
aclCheck,
graphAnalytics,
project => projections.statistics(project, None, GraphAnalytics.projectionName(project)),
schemeDirectives
schemeDirectives,
viewsQuery
)(
baseUri,
s,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics

import akka.http.scaladsl.model.Uri
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewRejection
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewRejection.WrappedElasticSearchClientError
import ch.epfl.bluebrain.nexus.delta.sdk.model.search.SortList
import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
import io.circe.{Json, JsonObject}
import monix.bio.IO

/** Allows to perform elasticsearch queries on Graph Analytics views */
trait GraphAnalyticsViewsQuery {

/**
* In a given project, perform the provided elasticsearch query on the projects' Graph Analytics view.
* @param projectRef
* project in which to make the query
* @param query
* elasticsearch query to perform on the Graph Analytics view
* @param qp
* the extra query parameters for the elasticsearch index
*/
def query(projectRef: ProjectRef, query: JsonObject, qp: Uri.Query): IO[ElasticSearchViewRejection, Json]
}

/**
* A [[GraphAnalyticsViewsQuery]] implementation that uses the [[ElasticSearchClient]] to query views.
* @param prefix
* prefix used in the names of the elasticsearch indices
* @param client
* elasticsearch client
*/
class GraphAnalyticsViewsQueryImpl(prefix: String, client: ElasticSearchClient) extends GraphAnalyticsViewsQuery {
override def query(projectRef: ProjectRef, query: JsonObject, qp: Uri.Query): IO[ElasticSearchViewRejection, Json] = {
val index = GraphAnalytics.index(prefix, projectRef)
client.search(query, Set(index.value), qp)(SortList.empty).mapError(WrappedElasticSearchClientError)
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,15 @@ package ch.epfl.bluebrain.nexus.delta.plugins.graph

import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts => nxvContexts}
import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission

package object analytics {
object contexts {
val relationships: Iri = nxvContexts + "relationships.json"
val properties: Iri = nxvContexts + "properties.json"
}

object permissions {
final val query: Permission = Permission.unsafe("views/query")
}
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.routes

import akka.http.scaladsl.server.Directives.{concat, get, pathEndOrSingleSlash, pathPrefix}
import akka.http.scaladsl.server.Directives.{as, concat, entity, get, pathEndOrSingleSlash, pathPrefix, post}
import akka.http.scaladsl.server.Route
import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.GraphAnalytics
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.routes.ElasticSearchViewsDirectives.extractQueryParams
import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.permissions.query
import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.{GraphAnalytics, GraphAnalyticsViewsQuery}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution
import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering
import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
Expand All @@ -15,7 +17,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri
import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources.{read => Read}
import ch.epfl.bluebrain.nexus.delta.sourcing.ProgressStatistics
import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
import kamon.instrumentation.akka.http.TracingDirectives.operationName
import io.circe.JsonObject
import monix.bio.UIO
import monix.execution.Scheduler

Expand All @@ -38,41 +40,49 @@ class GraphAnalyticsRoutes(
aclCheck: AclCheck,
graphAnalytics: GraphAnalytics,
fetchStatistics: ProjectRef => UIO[ProgressStatistics],
schemeDirectives: DeltaSchemeDirectives
schemeDirectives: DeltaSchemeDirectives,
viewsQuery: GraphAnalyticsViewsQuery
)(implicit baseUri: BaseUri, s: Scheduler, cr: RemoteContextResolution, ordering: JsonKeyOrdering)
extends AuthDirectives(identities, aclCheck)
with CirceUnmarshalling
with RdfMarshalling {
import baseUri.prefixSegment
import schemeDirectives._

def routes: Route =
baseUriPrefix(baseUri.prefix) {
pathPrefix("graph-analytics") {
extractCaller { implicit caller =>
(get & resolveProjectRef) { projectRef =>
resolveProjectRef { projectRef =>
concat(
// Fetch relationships
(pathPrefix("relationships") & pathEndOrSingleSlash) {
operationName(s"$prefixSegment/graph-analytics/{org}/{project}/relationships") {
authorizeFor(projectRef, Read).apply {
emit(graphAnalytics.relationships(projectRef))
get {
concat(
// Fetch relationships
(pathPrefix("relationships") & pathEndOrSingleSlash) {
authorizeFor(projectRef, Read).apply {
emit(graphAnalytics.relationships(projectRef))
}
},
// Fetch properties for a type
(pathPrefix("properties") & idSegment & pathEndOrSingleSlash) { tpe =>
authorizeFor(projectRef, Read).apply {
emit(graphAnalytics.properties(projectRef, tpe))
}
},
// Fetch the statistics
(pathPrefix("statistics") & pathEndOrSingleSlash) {
authorizeFor(projectRef, Read).apply {
emit(fetchStatistics(projectRef))
}
}
}
},
// Fetch properties for a type
(pathPrefix("properties") & idSegment & pathEndOrSingleSlash) { tpe =>
operationName(s"$prefixSegment/graph-analytics/{org}/{project}/properties/{type}") {
authorizeFor(projectRef, Read).apply {
emit(graphAnalytics.properties(projectRef, tpe))
}
}
)
},
// Fetch the statistics
(pathPrefix("statistics") & get & pathEndOrSingleSlash) {
operationName(s"$prefixSegment/graph-analytics/{org}/{project}/statistics") {
authorizeFor(projectRef, Read).apply {
emit(fetchStatistics(projectRef))
post {
// Search a graph analytics view
(pathPrefix("_search") & pathEndOrSingleSlash) {
authorizeFor(projectRef, query).apply {
(extractQueryParams & entity(as[JsonObject])) { (qp, query) =>
emit(viewsQuery.query(projectRef, query, qp))
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.model.AnalyticsGrap
import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.model.GraphAnalyticsRejection.ProjectContextRejection
import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.model.PropertiesStatistics.Metadata
import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.model.{AnalyticsGraph, GraphAnalyticsRejection, PropertiesStatistics}
import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.{contexts, GraphAnalytics}
import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.{contexts, permissions, GraphAnalytics}
import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary
import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.schema
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution
Expand Down Expand Up @@ -78,14 +78,17 @@ class GraphAnalyticsRoutesSpec extends BaseRouteSpec with CancelAfterFailure {
)
}

private val viewQueryResponse = json"""{"key": "value"}"""

private lazy val routes =
Route.seal(
new GraphAnalyticsRoutes(
identities,
aclCheck,
graphAnalytics,
_ => UIO.pure(ProgressStatistics(0L, 0L, 0L, 10L, Some(Instant.EPOCH), None)),
DeltaSchemeDirectives.empty
DeltaSchemeDirectives.empty,
(_, _, _) => IO.pure(viewQueryResponse)
).routes
)

Expand Down Expand Up @@ -142,6 +145,27 @@ class GraphAnalyticsRoutesSpec extends BaseRouteSpec with CancelAfterFailure {
}
}

"querying" should {

val query = json"""{ "query": { "match_all": {} } }"""

"fail without authorization" in {
Post("/v1/graph-analytics/org/project/_search", query.toEntity) ~> asAlice ~> routes ~> check {
response.status shouldEqual StatusCodes.Forbidden
response.asJson shouldEqual jsonContentOf("errors/authorization-failed.json")
}
}

"succeed" in {
aclCheck.append(AclAddress.Root, alice -> Set(permissions.query)).accepted
Post("/v1/graph-analytics/org/project/_search", query.toEntity) ~> asAlice ~> routes ~> check {
response.status shouldEqual StatusCodes.OK
response.asJson shouldEqual viewQueryResponse
}
}

}

}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
{
"hits": {
"hits": [
{
"_id": "http://example.com/person",
"_index": "delta_ga_myorg_myproj",
"_score": 1.3121864,
"_source": {
"@id": "http://example.com/person",
"@type": "http://schema.org/Person",
"properties": [
{
"@id": "http://example.com/epfl",
"dataType": "object",
"isInArray": false,
"path": "http://schema.org/worksFor"
}
],
"references": [
{
"@id": "http://example.com/epfl",
"@type": [
"http://schema.org/EducationalOrganization"
],
"dataType": "object",
"found": true,
"isInArray": false,
"path": "http://schema.org/worksFor"
}
],
"relationships": [
{
"@id": "http://example.com/epfl",
"@type": [
"http://schema.org/EducationalOrganization"
],
"dataType": "object",
"isInArray": false,
"path": "http://schema.org/worksFor"
}
],
"_createdAt": "2023-08-08T15:49:14.081Z",
"_createdBy": {
"@type": "User",
"realm": "internal",
"subject": "delta"
},
"_deprecated": false,
"_project": "myorg/myproj",
"_rev": 1,
"_updatedAt": "2023-08-08T15:49:14.081Z",
"_updatedBy": {
"@type": "User",
"realm": "internal",
"subject": "delta"
}
}
}
],
"max_score": 1.3121864,
"total": {
"relation": "eq",
"value": 1
}
},
"timed_out": false,
"took": 0,
"_shards": {
"failed": 0,
"skipped": 0,
"successful": 1,
"total": 1
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
curl -XPOST \
-H "Content-Type: application/json" \
"http://localhost:8080/v1/graph-analytics/myorg/myproj/_search" -d \
'{
"query": {
"term": {
"@id": "https://example.com/person"
}
}
}'
19 changes: 19 additions & 0 deletions docs/src/main/paradox/docs/delta/api/graph-analytics-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,25 @@ Request
Response
: @@snip [fetched-progress.json](assets/graph-analytics/fetched-progress.json)

## Search

```
POST /v1/graph-analytics/{org_label}/{project_label}/_search
{...}
```

Search documents that are in a given project's Graph Analytics view.

The supported payload is defined on the @link:[ElasticSearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html#search-search-api-request-body){ open=new }.

**Example**

Request
: @@snip [ga-search.sh](assets/graph-analytics/ga-search.sh)

Response
: @@snip [ge-search.json](assets/graph-analytics/ga-search.json)

## Internals

In order to implement the described endpoints we needed a way to transform our data so that it would answer the desired questions in a performant manner.
Expand Down
6 changes: 6 additions & 0 deletions docs/src/main/paradox/docs/releases/v1.9-release-notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ Storages can no longer be created with credentials that would get stored:

These should instead be defined in the Delta configuration.

### Graph Analytics

The Elasticsearch views behind Graph Analytics can now be queried using the `_search` endpoint.

@ref:[More information](../delta/api/graph-analytics-api.md)

## Nexus Fusion

TODO
Expand Down
Loading