From c9641cfb2c194e49257482e73040017809b6d915 Mon Sep 17 00:00:00 2001 From: An Phi Date: Wed, 16 Oct 2024 16:03:45 -0400 Subject: [PATCH] datacube: use query server for storage (#3173) * datacube: use query server for storage * address review comments * datacube: standardize input to Lambda * datacube: reorganize and optimize some operations --- .../query/api/ApplicationQuery.java | 141 ++++++- .../query/api/DataCubeQueryStoreManager.java | 308 ++++++++++++++ .../query/model/DataCubeQuery.java | 23 +- .../api/TestDataCubeQueryStoreManager.java | 388 ++++++++++++++++++ .../legend-engine-repl-data-cube/pom.xml | 8 - .../repl/dataCube/server/DataCubeHelpers.java | 46 ++- .../repl/dataCube/server/REPLServer.java | 6 +- .../dataCube/server/REPLServerHelpers.java | 74 ++-- .../server/handler/DataCubeQueryBuilder.java | 54 +-- .../model/DataCubeGetBaseQueryResult.java | 6 +- ...beGetQueryCodeRelationReturnTypeInput.java | 5 +- ...taCubeGetQueryRelationReturnTypeInput.java | 1 + ...eGetValueSpecificationCodeBatchInput.java} | 4 +- ...taCubeGetValueSpecificationCodeInput.java} | 4 +- .../dataCube/server/model/DataCubeQuery.java | 12 +- ...ataCubeQueryConfigurationDeserializer.java | 32 -- .../server/model/DataCubeQuerySource.java | 32 -- .../model/DataCubeQueryTypeaheadInput.java | 5 +- .../repl/dataCube/TestDataCubeHelpers.java | 48 ++- pom.xml | 2 +- 20 files changed, 989 insertions(+), 210 deletions(-) create mode 100644 legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/DataCubeQueryStoreManager.java rename legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuerySourceREPLExecutedQuery.java => legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/model/DataCubeQuery.java (52%) create mode 100644 legend-engine-application-query/src/test/java/org/finos/legend/engine/application/query/api/TestDataCubeQueryStoreManager.java rename legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/{DataCubeGetQueryCodeBatchInput.java => DataCubeGetValueSpecificationCodeBatchInput.java} (88%) rename legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/{DataCubeGetQueryCodeInput.java => DataCubeGetValueSpecificationCodeInput.java} (89%) delete mode 100644 legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQueryConfigurationDeserializer.java delete mode 100644 legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuerySource.java diff --git a/legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/ApplicationQuery.java b/legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/ApplicationQuery.java index fadb62d53f4..26de6a5607a 100644 --- a/legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/ApplicationQuery.java +++ b/legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/ApplicationQuery.java @@ -21,6 +21,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.eclipse.collections.api.list.MutableList; +import org.finos.legend.engine.application.query.model.DataCubeQuery; import org.finos.legend.engine.application.query.model.Query; import org.finos.legend.engine.application.query.model.QueryEvent; import org.finos.legend.engine.application.query.model.QuerySearchSpecification; @@ -32,15 +33,7 @@ import org.pac4j.core.profile.ProfileManager; import org.pac4j.jax.rs.annotations.Pac4JProfileManager; -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.util.List; @@ -51,10 +44,12 @@ public class ApplicationQuery { private final QueryStoreManager queryStoreManager; + private final DataCubeQueryStoreManager dataCubeQueryStoreManager; public ApplicationQuery(MongoClient mongoClient) { this.queryStoreManager = new QueryStoreManager(mongoClient); + this.dataCubeQueryStoreManager = new DataCubeQueryStoreManager(mongoClient); } private static String getCurrentUser(ProfileManager profileManager) @@ -255,4 +250,132 @@ public Response getQueryEvents(@QueryParam("queryId") @ApiParam("The query ID th return ExceptionTool.exceptionManager(e, LoggingEventType.GET_QUERY_EVENTS_ERROR, null); } } + + + @POST + @Path("dataCube/search") + @ApiOperation(value = "Search DataCube queries") + @Consumes({MediaType.APPLICATION_JSON}) + public Response searchDataCubeQueries(QuerySearchSpecification searchSpecification, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager profileManager) + { + try + { + return Response.ok().entity(this.dataCubeQueryStoreManager.searchQueries(searchSpecification, getCurrentUser(profileManager))).build(); + } + catch (Exception e) + { + if (e instanceof ApplicationQueryException) + { + return ((ApplicationQueryException) e).toResponse(); + } + return ExceptionTool.exceptionManager(e, LoggingEventType.SEARCH_QUERIES_ERROR, null); + } + } + + @GET + @Path("dataCube/batch") + @ApiOperation(value = "Get the DataCube queries with specified IDs") + @Consumes({MediaType.APPLICATION_JSON}) + public Response getDataCubeQueries(@QueryParam("queryIds") @ApiParam("The list of query IDs to fetch (must contain no more than 50 items)") List queryIds) + { + try + { + return Response.ok(this.dataCubeQueryStoreManager.getQueries(queryIds)).build(); + } + catch (Exception e) + { + if (e instanceof ApplicationQueryException) + { + return ((ApplicationQueryException) e).toResponse(); + } + return ExceptionTool.exceptionManager(e, LoggingEventType.GET_QUERIES_ERROR, null); + } + } + + @GET + @Path("dataCube/{queryId}") + @ApiOperation(value = "Get the DataCube query with specified ID") + @Consumes({MediaType.APPLICATION_JSON}) + public Response getDataCubeQuery(@PathParam("queryId") String queryId) + { + try + { + return Response.ok(this.dataCubeQueryStoreManager.getQuery(queryId)).build(); + } + catch (Exception e) + { + if (e instanceof ApplicationQueryException) + { + return ((ApplicationQueryException) e).toResponse(); + } + return ExceptionTool.exceptionManager(e, LoggingEventType.GET_QUERY_ERROR, null); + } + } + + @POST + @Path("dataCube") + @ApiOperation(value = "Create a new DataCube query") + @Consumes({MediaType.APPLICATION_JSON}) + public Response createDataCubeQuery(DataCubeQuery query, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager profileManager) + { + MutableList profiles = ProfileManagerHelper.extractProfiles(profileManager); + Identity identity = Identity.makeIdentity(profiles); + try + { + return Response.ok().entity(this.dataCubeQueryStoreManager.createQuery(query, getCurrentUser(profileManager))).build(); + } + catch (Exception e) + { + if (e instanceof ApplicationQueryException) + { + return ((ApplicationQueryException) e).toResponse(); + } + return ExceptionTool.exceptionManager(e, LoggingEventType.CREATE_QUERY_ERROR, identity.getName()); + } + } + + @PUT + @Path("dataCube/{queryId}") + @ApiOperation(value = "Update DataCube query") + @Consumes({MediaType.APPLICATION_JSON}) + public Response updateDataCubeQuery(@PathParam("queryId") String queryId, DataCubeQuery query, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager profileManager) + { + MutableList profiles = ProfileManagerHelper.extractProfiles(profileManager); + Identity identity = Identity.makeIdentity(profiles); + try + { + return Response.ok().entity(this.dataCubeQueryStoreManager.updateQuery(queryId, query, getCurrentUser(profileManager))).build(); + } + catch (Exception e) + { + if (e instanceof ApplicationQueryException) + { + return ((ApplicationQueryException) e).toResponse(); + } + return ExceptionTool.exceptionManager(e, LoggingEventType.UPDATE_QUERY_ERROR, identity.getName()); + } + } + + @DELETE + @Path("dataCube/{queryId}") + @ApiOperation(value = "Delete the DataCube query with specified ID") + @Consumes({MediaType.APPLICATION_JSON}) + public Response deleteDataCubeQuery(@PathParam("queryId") String queryId, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager profileManager) + { + MutableList profiles = ProfileManagerHelper.extractProfiles(profileManager); + Identity identity = Identity.makeIdentity(profiles); + try + { + this.dataCubeQueryStoreManager.deleteQuery(queryId, getCurrentUser(profileManager)); + return Response.noContent().build(); + } + catch (Exception e) + { + if (e instanceof ApplicationQueryException) + { + return ((ApplicationQueryException) e).toResponse(); + } + return ExceptionTool.exceptionManager(e, LoggingEventType.DELETE_QUERY_ERROR, identity.getName()); + } + } } diff --git a/legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/DataCubeQueryStoreManager.java b/legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/DataCubeQueryStoreManager.java new file mode 100644 index 00000000000..0cee796f942 --- /dev/null +++ b/legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/DataCubeQueryStoreManager.java @@ -0,0 +1,308 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.application.query.api; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.*; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.eclipse.collections.api.factory.SortedSets; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.set.sorted.MutableSortedSet; +import org.eclipse.collections.impl.utility.LazyIterate; +import org.finos.legend.engine.application.query.model.DataCubeQuery; +import org.finos.legend.engine.application.query.model.QuerySearchSortBy; +import org.finos.legend.engine.application.query.model.QuerySearchSpecification; +import org.finos.legend.engine.application.query.model.QuerySearchTermSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; +import org.finos.legend.engine.shared.core.vault.Vault; + +import javax.ws.rs.core.Response; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + +public class DataCubeQueryStoreManager +{ + private static final int MAX_NUMBER_OF_QUERIES = 100; + + private final ObjectMapper objectMapper = new ObjectMapper(); + private static final Document EMPTY_FILTER = Document.parse("{}"); + + private static final List LIGHT_QUERY_PROJECTION = Arrays.asList("id", "name", "createdAt", "lastUpdatedAt", "lastOpenAt"); + private static final int GET_QUERIES_LIMIT = 50; + + private final MongoClient mongoClient; + + public DataCubeQueryStoreManager(MongoClient mongoClient) + { + this.mongoClient = mongoClient; + } + + private MongoDatabase getDataCubeQueryDatabase() + { + if (Vault.INSTANCE.hasValue("query.mongo.database")) + { + return this.mongoClient.getDatabase(Vault.INSTANCE.getValue("query.mongo.database")); + } + throw new RuntimeException("DataCube Query MongoDB database has not been configured properly"); + } + + private MongoCollection getQueryCollection() + { + if (Vault.INSTANCE.hasValue("query.mongo.collection.dataCube")) + { + return this.getDataCubeQueryDatabase().getCollection(Vault.INSTANCE.getValue("query.mongo.collection.dataCube")); + } + throw new RuntimeException("DataCube Query MongoDB collection has not been configured properly"); + } + + private T documentToClass(Document document, Class _class) + { + try + { + return this.objectMapper.convertValue(document, _class); + } + catch (Exception e) + { + throw new ApplicationQueryException("Unable to deserialize document to class '" + _class.getName() + "':" + e.getMessage(), Response.Status.NOT_FOUND); + } + } + + private DataCubeQuery documentToQuery(Document document) + { + return this.documentToClass(document, DataCubeQuery.class); + } + + private Document queryToDocument(DataCubeQuery query) throws JsonProcessingException + { + return Document.parse(objectMapper.writeValueAsString(query)); + } + + private static void validate(boolean predicate, String message) + { + if (!predicate) + { + throw new ApplicationQueryException(message, Response.Status.BAD_REQUEST); + } + } + + private static void validateNonEmptyQueryField(String fieldValue, String message) + { + validate(fieldValue != null && !fieldValue.isEmpty(), message); + } + + private static void validateNonNullQueryField(Object fieldValue, String message) + { + validate(fieldValue != null, message); + } + + public static void validateQuery(DataCubeQuery query) + { + validateNonEmptyQueryField(query.id, "Query ID is missing or empty"); + validateNonEmptyQueryField(query.name, "Query name is missing or empty"); + validateNonNullQueryField(query.query, "Query is missing"); + validateNonNullQueryField(query.source, "Query source is missing"); + } + + public List searchQueries(QuerySearchSpecification searchSpecification, String currentUser) + { + List filters = new ArrayList<>(); + if (searchSpecification.searchTermSpecification != null) + { + QuerySearchTermSpecification querySearchTermSpecification = searchSpecification.searchTermSpecification; + if (querySearchTermSpecification.searchTerm == null) + { + throw new ApplicationQueryException("Query search spec expecting a search term", Response.Status.INTERNAL_SERVER_ERROR); + } + if (querySearchTermSpecification.exactMatchName != null && querySearchTermSpecification.exactMatchName) + { + Bson filter = Filters.eq("name", querySearchTermSpecification.searchTerm); + filters.add(filter); + } + else + { + Bson idFilter = Filters.eq("id", querySearchTermSpecification.searchTerm); + Bson nameFilter = Filters.regex("name", Pattern.quote(querySearchTermSpecification.searchTerm), "i"); + Bson filter = Filters.or(idFilter, nameFilter); + filters.add(filter); + } + } + + List queries = new ArrayList<>(); + List aggregateLists = new ArrayList<>(); + aggregateLists.add(Aggregates.match(filters.isEmpty() ? EMPTY_FILTER : Filters.and(filters))); + if (searchSpecification.sortByOption != null) + { + aggregateLists.add(Aggregates.sort(Sorts.descending(getSortByField(searchSpecification.sortByOption)))); + } + aggregateLists.add(Aggregates.project(Projections.include(LIGHT_QUERY_PROJECTION))); + aggregateLists.add(Aggregates.limit(Math.min(MAX_NUMBER_OF_QUERIES, searchSpecification.limit == null ? Integer.MAX_VALUE : searchSpecification.limit))); + AggregateIterable documents = this.getQueryCollection() + .aggregate(aggregateLists); + + for (Document doc : documents) + { + queries.add(documentToQuery(doc)); + } + return queries; + } + + public String getSortByField(QuerySearchSortBy sortBy) + { + switch (sortBy) + { + case SORT_BY_CREATE: + { + return "createdAt"; + } + case SORT_BY_VIEW: + { + return "lastOpenAt"; + } + case SORT_BY_UPDATE: + { + return "lastUpdatedAt"; + } + default: + { + throw new EngineException("Unknown sort-by value", EngineErrorType.COMPILATION); + } + } + } + + public List getQueries(List queryIds) + { + if (queryIds.size() > GET_QUERIES_LIMIT) + { + throw new ApplicationQueryException("Can't fetch more than " + GET_QUERIES_LIMIT + " queries", Response.Status.BAD_REQUEST); + } + MutableList matchingQueries = LazyIterate.collect(this.getQueryCollection().find(Filters.in("id", queryIds)).limit(GET_QUERIES_LIMIT), this::documentToQuery).toList(); + // validate + MutableSortedSet notFoundQueries = SortedSets.mutable.empty(); + MutableSortedSet duplicatedQueries = SortedSets.mutable.empty(); + queryIds.forEach(queryId -> + { + int count = matchingQueries.count(query -> queryId.equals(query.id)); + if (count > 1) + { + duplicatedQueries.add(queryId); + } + else if (count == 0) + { + notFoundQueries.add(queryId); + } + }); + if (!duplicatedQueries.isEmpty()) + { + throw new IllegalStateException(duplicatedQueries.makeString("Found multiple queries with duplicated ID for the following ID(s):\\n", "\\n", "")); + } + if (!notFoundQueries.isEmpty()) + { + throw new ApplicationQueryException(notFoundQueries.makeString("Can't find queries for the following ID(s):\\n", "\\n", ""), Response.Status.INTERNAL_SERVER_ERROR); + } + return matchingQueries; + } + + public DataCubeQuery getQuery(String queryId) throws JsonProcessingException + { + List matchingQueries = LazyIterate.collect(this.getQueryCollection().find(Filters.eq("id", queryId)), this::documentToQuery).toList(); + if (matchingQueries.size() > 1) + { + throw new IllegalStateException("Found multiple queries with ID '" + queryId + "'"); + } + else if (matchingQueries.isEmpty()) + { + throw new ApplicationQueryException("Can't find query with ID '" + queryId + "'", Response.Status.NOT_FOUND); + } + DataCubeQuery query = matchingQueries.get(0); + query.lastOpenAt = Instant.now().toEpochMilli(); + this.getQueryCollection().updateOne( + Filters.eq("id", queryId), + Updates.set("lastOpenAt", Instant.now().toEpochMilli()) + ); + return query; + } + + + public DataCubeQuery createQuery(DataCubeQuery query, String currentUser) throws JsonProcessingException + { + validateQuery(query); + // TODO: store ownership information + + List matchingQueries = LazyIterate.collect(this.getQueryCollection().find(Filters.eq("id", query.id)), this::documentToQuery).toList(); + if (!matchingQueries.isEmpty()) + { + throw new ApplicationQueryException("Query with ID '" + query.id + "' already existed", Response.Status.BAD_REQUEST); + } + query.createdAt = Instant.now().toEpochMilli(); + query.lastUpdatedAt = query.createdAt; + query.lastOpenAt = query.createdAt; + this.getQueryCollection().insertOne(queryToDocument(query)); + return query; + } + + public DataCubeQuery updateQuery(String queryId, DataCubeQuery query, String currentUser) throws JsonProcessingException + { + validateQuery(query); + + List matchingQueries = LazyIterate.collect(this.getQueryCollection().find(Filters.eq("id", queryId)), this::documentToQuery).toList(); + if (!queryId.equals(query.id)) + { + throw new ApplicationQueryException("Updating query ID is not supported", Response.Status.BAD_REQUEST); + } + if (matchingQueries.size() > 1) + { + throw new IllegalStateException("Found multiple queries with ID '" + queryId + "'"); + } + else if (matchingQueries.isEmpty()) + { + throw new ApplicationQueryException("Can't find query with ID '" + queryId + "'", Response.Status.NOT_FOUND); + } + DataCubeQuery currentQuery = matchingQueries.get(0); + + // TODO: check ownership + query.createdAt = currentQuery.createdAt; + query.lastUpdatedAt = Instant.now().toEpochMilli(); + query.lastOpenAt = Instant.now().toEpochMilli(); + this.getQueryCollection().findOneAndReplace(Filters.eq("id", queryId), queryToDocument(query)); + return query; + } + + public void deleteQuery(String queryId, String currentUser) throws JsonProcessingException + { + List matchingQueries = LazyIterate.collect(this.getQueryCollection().find(Filters.eq("id", queryId)), this::documentToQuery).toList(); + if (matchingQueries.size() > 1) + { + throw new IllegalStateException("Found multiple queries with ID '" + queryId + "'"); + } + else if (matchingQueries.isEmpty()) + { + throw new ApplicationQueryException("Can't find query with ID '" + queryId + "'", Response.Status.NOT_FOUND); + } + DataCubeQuery currentQuery = matchingQueries.get(0); + + // TODO: check ownership + this.getQueryCollection().findOneAndDelete(Filters.eq("id", queryId)); + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuerySourceREPLExecutedQuery.java b/legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/model/DataCubeQuery.java similarity index 52% rename from legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuerySourceREPLExecutedQuery.java rename to legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/model/DataCubeQuery.java index b8c36b1cdf1..dd277601d85 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuerySourceREPLExecutedQuery.java +++ b/legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/model/DataCubeQuery.java @@ -1,4 +1,4 @@ -// Copyright 2024 Goldman Sachs +// Copyright 2020 Goldman Sachs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.engine.repl.dataCube.server.model; +package org.finos.legend.engine.application.query.model; -public class DataCubeQuerySourceREPLExecutedQuery extends DataCubeQuerySource +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.Map; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataCubeQuery { + public String id; + public String name; + public String description; + + public Map query; + public Map source; + + public Long lastUpdatedAt; + public Long createdAt; + public Long lastOpenAt; + + // TODO: ownership } diff --git a/legend-engine-application-query/src/test/java/org/finos/legend/engine/application/query/api/TestDataCubeQueryStoreManager.java b/legend-engine-application-query/src/test/java/org/finos/legend/engine/application/query/api/TestDataCubeQueryStoreManager.java new file mode 100644 index 00000000000..5660f64c7a5 --- /dev/null +++ b/legend-engine-application-query/src/test/java/org/finos/legend/engine/application/query/api/TestDataCubeQueryStoreManager.java @@ -0,0 +1,388 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.application.query.api; + +import org.eclipse.collections.api.block.function.Function0; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; +import org.finos.legend.engine.application.query.model.DataCubeQuery; +import org.finos.legend.engine.application.query.model.QuerySearchSortBy; +import org.finos.legend.engine.application.query.model.QuerySearchSpecification; +import org.finos.legend.engine.application.query.model.QuerySearchTermSpecification; +import org.finos.legend.engine.application.query.utils.TestMongoClientProvider; +import org.finos.legend.engine.shared.core.vault.TestVaultImplementation; +import org.finos.legend.engine.shared.core.vault.Vault; +import org.junit.*; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class TestDataCubeQueryStoreManager +{ + static class TestDataCubeQuerySearchSpecificationBuilder + { + public QuerySearchTermSpecification searchTermSpecification; + public Integer limit; + public Boolean showCurrentUserQueriesOnly; + public QuerySearchSortBy sortByOption; + + TestDataCubeQuerySearchSpecificationBuilder withSearchTerm(String searchTerm) + { + if (this.searchTermSpecification == null) + { + this.searchTermSpecification = new QuerySearchTermSpecification(); + } + this.searchTermSpecification.searchTerm = searchTerm; + return this; + } + + TestDataCubeQuerySearchSpecificationBuilder withLimit(Integer limit) + { + this.limit = limit; + return this; + } + + TestDataCubeQuerySearchSpecificationBuilder withShowCurrentUserQueriesOnly(Boolean showCurrentUserQueriesOnly) + { + this.showCurrentUserQueriesOnly = showCurrentUserQueriesOnly; + return this; + } + + TestDataCubeQuerySearchSpecificationBuilder withExactNameSearch(Boolean exactMatchName) + { + if (this.searchTermSpecification == null) + { + this.searchTermSpecification = new QuerySearchTermSpecification(); + } + this.searchTermSpecification.exactMatchName = exactMatchName; + return this; + } + + TestDataCubeQuerySearchSpecificationBuilder withSortByOption(QuerySearchSortBy sortByOption) + { + this.sortByOption = sortByOption; + return this; + } + + QuerySearchSpecification build() + { + QuerySearchSpecification searchSpecification = new QuerySearchSpecification(); + searchSpecification.searchTermSpecification = this.searchTermSpecification; + searchSpecification.limit = this.limit; + searchSpecification.showCurrentUserQueriesOnly = this.showCurrentUserQueriesOnly; + searchSpecification.sortByOption = this.sortByOption; + return searchSpecification; + } + } + + static class TestQueryBuilder + { + public String id; + public String name; + public String description = "description"; + public Map query = Maps.mutable.empty(); + public Map source = Maps.mutable.empty(); + + static TestQueryBuilder create(String id, String name) + { + TestQueryBuilder queryBuilder = new TestQueryBuilder(); + queryBuilder.id = id; + queryBuilder.name = name; + return queryBuilder; + } + + DataCubeQuery build() + { + DataCubeQuery query = new DataCubeQuery(); + query.id = this.id; + query.name = this.name; + query.description = this.description; + query.query = this.query; + query.source = this.source; + return query; + } + } + + private TestMongoClientProvider testMongoClientProvider = new TestMongoClientProvider(); + private final DataCubeQueryStoreManager storeManager = new DataCubeQueryStoreManager(testMongoClientProvider.mongoClient); + private static final TestVaultImplementation testVaultImplementation = new TestVaultImplementation(); + + @BeforeClass + public static void setupClass() + { + testVaultImplementation.setValue("query.mongo.database", "test"); + testVaultImplementation.setValue("query.mongo.collection.dataCube", "dataCube"); + Vault.INSTANCE.registerImplementation(testVaultImplementation); + } + + @AfterClass + public static void cleanUpClass() + { + Vault.INSTANCE.unregisterImplementation(testVaultImplementation); + } + + @Before + public void setup() + { + this.testMongoClientProvider = new TestMongoClientProvider(); + } + + @After + public void cleanUp() + { + this.testMongoClientProvider.cleanUp(); + } + + @Test + public void testValidateQuery() + { + Function0 _createTestQuery = () -> TestQueryBuilder.create("1", "query1").build(); + DataCubeQuery goodQuery = _createTestQuery.get(); + DataCubeQueryStoreManager.validateQuery(goodQuery); + + // ID + DataCubeQuery queryWithInvalidId = _createTestQuery.get(); + queryWithInvalidId.id = null; + Assert.assertEquals("Query ID is missing or empty", Assert.assertThrows(ApplicationQueryException.class, () -> DataCubeQueryStoreManager.validateQuery(queryWithInvalidId)).getMessage()); + queryWithInvalidId.id = ""; + Assert.assertEquals("Query ID is missing or empty", Assert.assertThrows(ApplicationQueryException.class, () -> DataCubeQueryStoreManager.validateQuery(queryWithInvalidId)).getMessage()); + + // Name + DataCubeQuery queryWithInvalidName = _createTestQuery.get(); + queryWithInvalidName.name = null; + Assert.assertEquals("Query name is missing or empty", Assert.assertThrows(ApplicationQueryException.class, () -> DataCubeQueryStoreManager.validateQuery(queryWithInvalidName)).getMessage()); + queryWithInvalidId.name = ""; + Assert.assertEquals("Query name is missing or empty", Assert.assertThrows(ApplicationQueryException.class, () -> DataCubeQueryStoreManager.validateQuery(queryWithInvalidName)).getMessage()); + + // TODO?: validate content + } + + @Test + public void testSearchQueries() throws Exception + { + String currentUser = "testUser"; + DataCubeQuery newQuery = TestQueryBuilder.create("1", "query1").build(); + storeManager.createQuery(newQuery, currentUser); + List queries = storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().build(), currentUser); + Assert.assertEquals(1, queries.size()); + DataCubeQuery lightQuery = queries.get(0); + Assert.assertEquals("1", lightQuery.id); + Assert.assertEquals("query1", lightQuery.name); + Assert.assertNotNull(lightQuery.createdAt); + Assert.assertNotNull(lightQuery.lastUpdatedAt); + Assert.assertNull(lightQuery.description); + } + + @Test + public void testMatchExactNameQuery() throws Exception + { + String currentUser = "testUser"; + DataCubeQuery newQuery = TestQueryBuilder.create("1", "Test Query 1").build(); + DataCubeQuery newQueryTwo = TestQueryBuilder.create("2", "Test Query 12").build(); + DataCubeQuery newQueryThree = TestQueryBuilder.create("3", "Test Query 13").build(); + storeManager.createQuery(newQuery, currentUser); + storeManager.createQuery(newQueryTwo, currentUser); + storeManager.createQuery(newQueryThree, currentUser); + List queriesGeneralSearch = storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSearchTerm("Test Query 1").build(), currentUser); + Assert.assertEquals(3, queriesGeneralSearch.size()); + List queriesExactSearch = storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSearchTerm("Test Query 1").withExactNameSearch(true).build(), currentUser); + Assert.assertEquals(1, queriesExactSearch.size()); + } + + @Test + public void testGetQueriesWithLimit() throws Exception + { + String currentUser = "testUser"; + storeManager.createQuery(TestQueryBuilder.create("1", "query1").build(), currentUser); + storeManager.createQuery(TestQueryBuilder.create("2", "query2").build(), currentUser); + Assert.assertEquals(1, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withLimit(1).build(), currentUser).size()); + Assert.assertEquals(2, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().build(), currentUser).size()); + Assert.assertEquals(0, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withLimit(0).build(), currentUser).size()); + } + + @Test + public void testGetQueriesWithSortBy() throws Exception + { + String currentUser = "testUser"; + DataCubeQuery testQuery1 = TestQueryBuilder.create("1", "query1").build(); + DataCubeQuery testQuery2 = TestQueryBuilder.create("2", "query2").build(); + DataCubeQuery testQuery3 = TestQueryBuilder.create("3", "query3").build(); + DataCubeQuery testQuery4 = TestQueryBuilder.create("4", "query4").build(); + + // create in order 1 -> 4 -> 2 -> 3 + storeManager.createQuery(testQuery1, currentUser); + Thread.sleep(100); + storeManager.createQuery(testQuery4, currentUser); + Thread.sleep(100); + storeManager.createQuery(testQuery2, currentUser); + Thread.sleep(100); + storeManager.createQuery(testQuery3, currentUser); + + Assert.assertEquals(4, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSortByOption(QuerySearchSortBy.SORT_BY_CREATE).build(), currentUser).size()); + Assert.assertEquals(Arrays.asList("3", "2", "4", "1"), storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSortByOption(QuerySearchSortBy.SORT_BY_CREATE).build(), currentUser).stream().map(q -> q.id).collect(Collectors.toList())); + + storeManager.updateQuery("2", TestQueryBuilder.create("2", "query2NewlyUpdated").build(), currentUser); + Assert.assertEquals(Arrays.asList("2", "3", "4", "1"), storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSortByOption(QuerySearchSortBy.SORT_BY_UPDATE).build(), currentUser).stream().map(q -> q.id).collect(Collectors.toList())); + + storeManager.getQuery("1"); + Assert.assertEquals(Arrays.asList("1", "2", "3", "4"), storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSortByOption(QuerySearchSortBy.SORT_BY_VIEW).build(), currentUser).stream().map(q -> q.id).collect(Collectors.toList())); + } + + @Test + public void testGetQueriesWithSearchText() throws Exception + { + String currentUser = "testUser"; + storeManager.createQuery(TestQueryBuilder.create("1", "query1").build(), currentUser); + storeManager.createQuery(TestQueryBuilder.create("2", "query2").build(), currentUser); + storeManager.createQuery(TestQueryBuilder.create("3", "query2").build(), currentUser); + Assert.assertEquals(3, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().build(), currentUser).size()); + Assert.assertEquals(1, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSearchTerm("query1").build(), currentUser).size()); + Assert.assertEquals(2, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSearchTerm("query2").build(), currentUser).size()); + Assert.assertEquals(3, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSearchTerm("query").build(), currentUser).size()); + } + + @Test + public void testGetNotFoundQuery() + { + Assert.assertEquals("Can't find query with ID '1'", Assert.assertThrows(ApplicationQueryException.class, () -> storeManager.getQuery("1")).getMessage()); + } + + @Test + public void testCreateSimpleQuery() throws Exception + { + String currentUser = "testUser"; + DataCubeQuery newQuery = TestQueryBuilder.create("1", "query1").build(); + DataCubeQuery createdQuery = storeManager.createQuery(newQuery, currentUser); + Assert.assertEquals("1", createdQuery.id); + Assert.assertEquals("query1", createdQuery.name); + Assert.assertEquals("description", createdQuery.description); + Assert.assertNotNull(createdQuery.createdAt); + Assert.assertEquals(createdQuery.createdAt, createdQuery.lastUpdatedAt); + Assert.assertEquals(createdQuery.lastOpenAt, createdQuery.lastUpdatedAt); + } + + @Test + public void testCreateInvalidQuery() + { + String currentUser = "testUser"; + Assert.assertEquals("Query name is missing or empty", Assert.assertThrows(ApplicationQueryException.class, () -> storeManager.createQuery(TestQueryBuilder.create("1", null).build(), currentUser)).getMessage()); + } + + @Test + public void testCreateQueryWithSameId() throws Exception + { + String currentUser = "testUser"; + storeManager.createQuery(TestQueryBuilder.create("1", "query1").build(), currentUser); + Assert.assertEquals("Query with ID '1' already existed", Assert.assertThrows(ApplicationQueryException.class, () -> storeManager.createQuery(TestQueryBuilder.create("1", "query1").build(), currentUser)).getMessage()); + } + + @Test + public void testUpdateQuery() throws Exception + { + String currentUser = "testUser"; + storeManager.createQuery(TestQueryBuilder.create("1", "query1").build(), currentUser); + storeManager.updateQuery("1", TestQueryBuilder.create("1", "query2").build(), currentUser); + Assert.assertEquals("query2", storeManager.getQuery("1").name); + } + + @Test + public void testUpdateWithInvalidQuery() + { + String currentUser = "testUser"; + Assert.assertEquals("Query name is missing or empty", Assert.assertThrows(ApplicationQueryException.class, () -> storeManager.updateQuery("1", TestQueryBuilder.create("1", null).build(), currentUser)).getMessage()); + } + + @Test + public void testUpdateNotFoundQuery() + { + String currentUser = "testUser"; + Assert.assertThrows(ApplicationQueryException.class, () -> storeManager.updateQuery("1", TestQueryBuilder.create("1", "query1").build(), currentUser)); + } + + @Test + public void testDeleteQuery() throws Exception + { + String currentUser = "testUser"; + storeManager.createQuery(TestQueryBuilder.create("1", "query1").build(), currentUser); + storeManager.deleteQuery("1", currentUser); + Assert.assertEquals(0, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().build(), currentUser).size()); + } + + @Test + public void testDeleteNotFoundQuery() + { + String currentUser = "testUser"; + Assert.assertEquals("Can't find query with ID '1'", Assert.assertThrows(ApplicationQueryException.class, () -> storeManager.deleteQuery("1", currentUser)).getMessage()); + } + + @Test + public void testCreateSimpleQueryContainsTimestamps() throws Exception + { + String currentUser = "testUser"; + DataCubeQuery newQuery = TestQueryBuilder.create("1", "query1").build(); + DataCubeQuery createdQuery = storeManager.createQuery(newQuery, currentUser); + Assert.assertNotNull(createdQuery.lastUpdatedAt); + Assert.assertNotNull(createdQuery.createdAt); + } + + @Test + public void testSearchQueriesContainTimestamps() throws Exception + { + String currentUser = "testUser"; + DataCubeQuery newQuery = TestQueryBuilder.create("1", "query1").build(); + storeManager.createQuery(newQuery, currentUser); + List queries = storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().build(), currentUser); + Assert.assertEquals(1, queries.size()); + DataCubeQuery lightQuery = queries.get(0); + Assert.assertNotNull(lightQuery.lastUpdatedAt); + Assert.assertNotNull(lightQuery.createdAt); + } + + @Test + public void testSearchQueriesWithSearchByQueryId() throws Exception + { + String currentUser = "testUser"; + storeManager.createQuery(TestQueryBuilder.create("26929514-237c-11ed-861d-0242ac120002", "query_a").build(), currentUser); + storeManager.createQuery(TestQueryBuilder.create("26929515-237c-11bd-851d-0243ac120002", "query_b").build(), currentUser); + storeManager.createQuery(TestQueryBuilder.create("23929515-235c-11ad-851d-0143ac120002", "query_c").build(), currentUser); + Assert.assertEquals(3, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().build(), currentUser).size()); + Assert.assertEquals(1, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSearchTerm("23929515-235c-11ad-851d-0143ac120002").build(), currentUser).size()); + Assert.assertEquals(0, storeManager.searchQueries(new TestDataCubeQuerySearchSpecificationBuilder().withSearchTerm("23929515-235c-11ad").build(), currentUser).size()); + } + + @Test + public void testGetQueries() throws Exception + { + String currentUser = "testUser"; + DataCubeQuery testQuery1 = TestQueryBuilder.create("1", "query1").build(); + DataCubeQuery testQuery2 = TestQueryBuilder.create("2", "query2").build(); + DataCubeQuery testQuery3 = TestQueryBuilder.create("3", "query3").build(); + storeManager.createQuery(testQuery1, currentUser); + storeManager.createQuery(testQuery2, currentUser); + storeManager.createQuery(testQuery3, currentUser); + + Assert.assertEquals(1, storeManager.getQueries(Lists.fixedSize.of("2")).size()); + Assert.assertEquals(1, storeManager.getQueries(Lists.fixedSize.of("3")).size()); + Assert.assertEquals(2, storeManager.getQueries(Lists.fixedSize.of("2", "3")).size()); + + Assert.assertEquals("Can't find queries for the following ID(s):\\n4", Assert.assertThrows(ApplicationQueryException.class, () -> storeManager.getQueries(Lists.fixedSize.of("4"))).getMessage()); + Assert.assertEquals("Can't find queries for the following ID(s):\\n4\\n6", Assert.assertThrows(ApplicationQueryException.class, () -> storeManager.getQueries(Lists.fixedSize.of("4", "3", "6"))).getMessage()); + + Assert.assertEquals("Can't fetch more than 50 queries", Assert.assertThrows(ApplicationQueryException.class, () -> storeManager.getQueries(Lists.fixedSize.ofAll(Collections.nCopies(51, "5")))).getMessage()); + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/pom.xml b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/pom.xml index a2edc86d401..cda5e6699ac 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/pom.xml +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/pom.xml @@ -203,18 +203,10 @@ eclipse-collections-api - - com.fasterxml.jackson.core - jackson-core - com.fasterxml.jackson.core jackson-databind - - com.fasterxml.jackson.core - jackson-annotations - org.jline diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/DataCubeHelpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/DataCubeHelpers.java index 3a7542c1cf3..6c802da0fa4 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/DataCubeHelpers.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/DataCubeHelpers.java @@ -35,6 +35,7 @@ import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity; import org.finos.legend.engine.protocol.pure.v1.model.relationType.RelationType; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; @@ -50,6 +51,7 @@ import org.finos.legend.engine.shared.core.kerberos.SubjectTools; import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan; import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; +import org.finos.legend.pure.m3.navigation.M3Paths; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -125,6 +127,20 @@ public static DataCubeExecutionResult executeQuery(Client client, LegendInterfac } } + public static RelationType getRelationReturnType(LegendInterface legendInterface, Lambda lambda, PureModelContextData data) + { + PureModelContextData pmcd; + if (data != null) + { + pmcd = DataCubeHelpers.injectNewFunction(data, lambda).getOne(); + } + else + { + pmcd = PureModelContextData.newBuilder().withElement(wrapLambda(lambda)).build(); + } + return getRelationReturnType(legendInterface, pmcd); + } + public static RelationType getRelationReturnType(LegendInterface legendInterface, PureModelContextData data) { PureModel pureModel = legendInterface.compile(data); @@ -141,16 +157,21 @@ public static String getQueryCode(ValueSpecification valueSpecification, Boolean return valueSpecification.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().withRenderStyle(pretty != null && pretty ? RenderStyle.PRETTY : RenderStyle.STANDARD).build()); } - public static CompletionResult getCodeTypeahead(String code, String baseQueryCode, PureModelContextData data, MutableList extensions, LegendInterface legendInterface) + public static CompletionResult getCodeTypeahead(String code, Lambda lambda, PureModelContextData data, MutableList extensions, LegendInterface legendInterface) { try { - PureModelContextData newData = PureModelContextData.newBuilder() - .withOrigin(data.getOrigin()) - .withSerializer(data.getSerializer()) - .withElements(ListIterate.select(data.getElements(), el -> !el.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH))) - .build(); - String graphCode = PureGrammarComposer.newInstance(PureGrammarComposerContext.Builder.newInstance().build()).renderPureModelContextData(newData); + String graphCode = ""; + if (data != null) + { + PureModelContextData newData = PureModelContextData.newBuilder() + .withOrigin(data.getOrigin()) + .withSerializer(data.getSerializer()) + .withElements(ListIterate.select(data.getElements(), el -> !el.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH))) + .build(); + graphCode = PureGrammarComposer.newInstance(PureGrammarComposerContext.Builder.newInstance().build()).renderPureModelContextData(newData); + } + String baseQueryCode = lambda != null ? getQueryCode(lambda.body.get(0), false) : null; String queryCode = (baseQueryCode != null ? baseQueryCode : "") + code; Completer completer = new Completer(graphCode, extensions, legendInterface); CompletionResult result = completer.complete(queryCode); @@ -166,6 +187,17 @@ public static CompletionResult getCodeTypeahead(String code, String baseQueryCod } } + public static Function wrapLambda(Lambda lambda) + { + Function func = new Function(); + func.name = REPL_RUN_FUNCTION_QUALIFIED_PATH.substring(REPL_RUN_FUNCTION_QUALIFIED_PATH.lastIndexOf("::") + 2); + func._package = REPL_RUN_FUNCTION_QUALIFIED_PATH.substring(0, REPL_RUN_FUNCTION_QUALIFIED_PATH.lastIndexOf("::")); + func.returnType = M3Paths.Any; + func.returnMultiplicity = new Multiplicity(0, null); + func.body = lambda.body; + return func; + } + /** * Replace the magic function in the given graph data by a new function with the body of the specified lambda */ diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServer.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServer.java index b72b2bd6f22..9ffcb8ffe1a 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServer.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServer.java @@ -81,9 +81,9 @@ public void initialize() throws Exception .withKeyValue("/repl/", new DataCubeInfrastructure.StaticContent()) .withKeyValue("/api/dataCube/infrastructureInfo", new DataCubeInfrastructure.GridLicenseKey()) .withKeyValue("/api/dataCube/typeahead", new DataCubeQueryBuilder.QueryTypeahead()) - .withKeyValue("/api/dataCube/parseQuery", new DataCubeQueryBuilder.ParseQuery()) - .withKeyValue("/api/dataCube/getQueryCode", new DataCubeQueryBuilder.GetQueryCode()) - .withKeyValue("/api/dataCube/getQueryCode/batch", new DataCubeQueryBuilder.GetQueryCodeBatch()) + .withKeyValue("/api/dataCube/parseValueSpecification", new DataCubeQueryBuilder.ParseValueSpecification()) + .withKeyValue("/api/dataCube/getValueSpecificationCode", new DataCubeQueryBuilder.GetValueSpecificationCode()) + .withKeyValue("/api/dataCube/getValueSpecificationCode/batch", new DataCubeQueryBuilder.GetValueSpecificationCodeBatch()) .withKeyValue("/api/dataCube/getBaseQuery", new DataCubeQueryBuilder.GetBaseQuery()) .withKeyValue("/api/dataCube/getRelationReturnType", new DataCubeQueryBuilder.GetRelationReturnType()) .withKeyValue("/api/dataCube/getRelationReturnType/code", new DataCubeQueryBuilder.GetQueryCodeRelationReturnType()) diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServerHelpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServerHelpers.java index 1dda19d267c..78f0ec301cc 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServerHelpers.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServerHelpers.java @@ -20,6 +20,7 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; @@ -41,7 +42,6 @@ import org.finos.legend.engine.repl.core.legend.LegendInterface; import org.finos.legend.engine.repl.dataCube.server.model.DataCubeQuery; import org.finos.legend.engine.repl.dataCube.server.model.DataCubeQueryColumn; -import org.finos.legend.engine.repl.dataCube.server.model.DataCubeQuerySourceREPLExecutedQuery; import org.finos.legend.engine.repl.shared.ExecutionHelper; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; import org.finos.legend.pure.m3.navigation.M3Paths; @@ -49,7 +49,10 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import static org.finos.legend.engine.repl.shared.ExecutionHelper.REPL_RUN_FUNCTION_QUALIFIED_PATH; @@ -71,29 +74,6 @@ public static void handleResponse(HttpExchange exchange, int responseCode, Strin } } - public static Map getQueryParams(HttpExchange exchange) - { - String query = exchange.getRequestURI().getQuery(); - Map result = new HashMap<>(); - if (query == null) - { - return result; - } - for (String param : query.split("&")) - { - String[] entry = param.split("="); - if (entry.length > 1) - { - result.put(entry[0], entry[1]); - } - else - { - result.put(entry[0], ""); - } - } - return result; - } - public static class REPLServerState { public final Client client; @@ -104,6 +84,7 @@ public static class REPLServerState private PureModelContextData currentPureModelContextData; private DataCubeQuery query; + private Map source; public REPLServerState(Client client, ObjectMapper objectMapper, PlanExecutor planExecutor, LegendInterface legendInterface) { @@ -116,16 +97,9 @@ public REPLServerState(Client client, ObjectMapper objectMapper, PlanExecutor pl private void initialize(PureModelContextData pureModelContextData, List columns) { this.currentPureModelContextData = pureModelContextData; - this.startTime = System.currentTimeMillis(); - this.query = new DataCubeQuery(); - this.query.name = "New Report"; - this.query.configuration = null; // initially, the config is not initialized - - // process source - DataCubeQuerySourceREPLExecutedQuery source = new DataCubeQuerySourceREPLExecutedQuery(); - source.columns = columns; + // -------------------- SOURCE -------------------- // try to extract the runtime for the query // remove any usage of multiple from(), only add one to the end // TODO: we might need to account for other variants of ->from(), such as when mapping is specified @@ -173,33 +147,30 @@ else if (fn.parameters.size() == 3) fn.parameters.set(0, currentExpression); currentExpression = fn; } + Map source = Maps.mutable.empty(); + source.put("_type", "repl"); + source.put("timestamp", this.startTime); + source.put("query", currentExpression.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build())); + source.put("runtime", runtime); + source.put("mapping", mapping); + this.source = source; - this.query.partialQuery = ""; - source.query = currentExpression.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build()); - source.runtime = runtime; - source.mapping = mapping; - this.query.source = source; - - // build the partial query + // -------------------- QUERY -------------------- + DataCubeQuery query = new DataCubeQuery(); + query.configuration = null; // initially, the config is not initialized // NOTE: for this, the initial query is going to be a select all AppliedFunction partialFn = new AppliedFunction(); partialFn.function = "select"; ColSpecArray colSpecArray = new ColSpecArray(); - colSpecArray.colSpecs = ListIterate.collect(source.columns, col -> + colSpecArray.colSpecs = ListIterate.collect(columns, col -> { ColSpec colSpec = new ColSpec(); colSpec.name = col.name; return colSpec; }); partialFn.parameters = Lists.mutable.with(new ClassInstance("colSpecArray", colSpecArray, null)); - this.query.partialQuery = partialFn.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build()); - - // build the full query - AppliedFunction fullFn = new AppliedFunction(); - fullFn.function = "from"; - fullFn.parameters = mapping != null ? Lists.mutable.with(partialFn, new PackageableElementPtr(mapping), new PackageableElementPtr(runtime)) : Lists.mutable.with(partialFn, new PackageableElementPtr(runtime)); - partialFn.parameters = Lists.mutable.with(currentExpression).withAll(partialFn.parameters); - this.query.query = fullFn.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build()); + query.query = partialFn.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build()); + this.query = query; } public void initializeFromTable(PureModelContextData pureModelContextData) @@ -270,6 +241,11 @@ public DataCubeQuery getQuery() { return this.query; } + + public Map getSource() + { + return this.source; + } } public interface DataCubeServerHandler diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/handler/DataCubeQueryBuilder.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/handler/DataCubeQueryBuilder.java index e516c8bb579..94833b616a0 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/handler/DataCubeQueryBuilder.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/handler/DataCubeQueryBuilder.java @@ -25,7 +25,6 @@ import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.relationType.RelationType; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; import org.finos.legend.engine.repl.autocomplete.CompletionResult; import org.finos.legend.engine.repl.dataCube.server.DataCubeHelpers; import org.finos.legend.engine.repl.dataCube.server.model.*; @@ -34,6 +33,7 @@ import java.io.BufferedReader; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.Map; import java.util.stream.Collectors; import static org.finos.legend.engine.repl.dataCube.server.REPLServerHelpers.*; @@ -42,7 +42,7 @@ public class DataCubeQueryBuilder { - public static class ParseQuery implements DataCubeServerHandler + public static class ParseValueSpecification implements DataCubeServerHandler { @Override public HttpHandler getHandler(REPLServerState state) @@ -69,7 +69,7 @@ public HttpHandler getHandler(REPLServerState state) } } - public static class GetQueryCode implements DataCubeServerHandler + public static class GetValueSpecificationCode implements DataCubeServerHandler { @Override public HttpHandler getHandler(REPLServerState state) @@ -83,8 +83,8 @@ public HttpHandler getHandler(REPLServerState state) InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); BufferedReader bufferReader = new BufferedReader(inputStreamReader); String requestBody = bufferReader.lines().collect(Collectors.joining()); - DataCubeGetQueryCodeInput input = state.objectMapper.readValue(requestBody, DataCubeGetQueryCodeInput.class); - handleResponse(exchange, 200, DataCubeHelpers.getQueryCode(input.query, input.pretty), state); + DataCubeGetValueSpecificationCodeInput input = state.objectMapper.readValue(requestBody, DataCubeGetValueSpecificationCodeInput.class); + handleResponse(exchange, 200, DataCubeHelpers.getQueryCode(input.value, input.pretty), state); } catch (Exception e) { @@ -95,7 +95,7 @@ public HttpHandler getHandler(REPLServerState state) } } - public static class GetQueryCodeBatch implements DataCubeServerHandler + public static class GetValueSpecificationCodeBatch implements DataCubeServerHandler { @Override public HttpHandler getHandler(REPLServerState state) @@ -109,9 +109,9 @@ public HttpHandler getHandler(REPLServerState state) InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); BufferedReader bufferReader = new BufferedReader(inputStreamReader); String requestBody = bufferReader.lines().collect(Collectors.joining()); - DataCubeGetQueryCodeBatchInput input = state.objectMapper.readValue(requestBody, DataCubeGetQueryCodeBatchInput.class); + DataCubeGetValueSpecificationCodeBatchInput input = state.objectMapper.readValue(requestBody, DataCubeGetValueSpecificationCodeBatchInput.class); DataCubeGetQueryCodeBatchResult result = new DataCubeGetQueryCodeBatchResult(); - MapAdapter.adapt(input.queries).forEachKeyValue((key, value) -> + MapAdapter.adapt(input.values).forEachKeyValue((key, value) -> { try { @@ -148,8 +148,7 @@ public HttpHandler getHandler(REPLServerState state) BufferedReader bufferReader = new BufferedReader(inputStreamReader); String requestBody = bufferReader.lines().collect(Collectors.joining()); DataCubeQueryTypeaheadInput input = state.objectMapper.readValue(requestBody, DataCubeQueryTypeaheadInput.class); - PureModelContextData data = state.getCurrentPureModelContextData(); - CompletionResult result = DataCubeHelpers.getCodeTypeahead(input.code, DataCubeHelpers.getQueryCode(input.baseQuery, false), data, state.client.getCompleterExtensions(), state.legendInterface); + CompletionResult result = DataCubeHelpers.getCodeTypeahead(input.code, input.baseQuery, input.isolated ? null : state.getCurrentPureModelContextData(), state.client.getCompleterExtensions(), state.legendInterface); handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result.getCompletion()), state); } catch (Exception e) @@ -176,9 +175,7 @@ public HttpHandler getHandler(REPLServerState state) BufferedReader bufferReader = new BufferedReader(inputStreamReader); String requestBody = bufferReader.lines().collect(Collectors.joining()); DataCubeGetQueryRelationReturnTypeInput input = state.objectMapper.readValue(requestBody, DataCubeGetQueryRelationReturnTypeInput.class); - Lambda lambda = input.query; // if no lambda is specified, we're executing the initial query - PureModelContextData data = DataCubeHelpers.injectNewFunction(state.getCurrentPureModelContextData(), lambda).getOne(); - handleResponse(exchange, 200, state.objectMapper.writeValueAsString(DataCubeHelpers.getRelationReturnType(state.legendInterface, data)), state); + handleResponse(exchange, 200, state.objectMapper.writeValueAsString(DataCubeHelpers.getRelationReturnType(state.legendInterface, input.query, input.isolated ? null : state.getCurrentPureModelContextData())), state); } catch (Exception e) { @@ -204,17 +201,21 @@ public HttpHandler getHandler(REPLServerState state) BufferedReader bufferReader = new BufferedReader(inputStreamReader); String requestBody = bufferReader.lines().collect(Collectors.joining()); DataCubeGetQueryCodeRelationReturnTypeInput input = state.objectMapper.readValue(requestBody, DataCubeGetQueryCodeRelationReturnTypeInput.class); - PureModelContextData currentData = state.getCurrentPureModelContextData(); - PureModelContextData newData = PureModelContextData.newBuilder() - .withOrigin(currentData.getOrigin()) - .withSerializer(currentData.getSerializer()) - .withElements(ListIterate.select(currentData.getElements(), el -> !el.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH))) - .build(); - String graphCode = PureGrammarComposer.newInstance(PureGrammarComposerContext.Builder.newInstance().build()).renderPureModelContextData(newData); - graphCode += "\n###Pure\n" + - "import meta::pure::functions::relation::*;\n" + - "function " + REPL_RUN_FUNCTION_SIGNATURE + "{\n"; - graphCode += DataCubeHelpers.getQueryCode(input.baseQuery, false) + "\n"; + + String graphCode = ""; + if (!input.isolated) + { + PureModelContextData currentData = state.getCurrentPureModelContextData(); + PureModelContextData newData = PureModelContextData.newBuilder() + .withOrigin(currentData.getOrigin()) + .withSerializer(currentData.getSerializer()) + .withElements(ListIterate.select(currentData.getElements(), el -> !el.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH))) + .build(); + graphCode += PureGrammarComposer.newInstance(PureGrammarComposerContext.Builder.newInstance().build()).renderPureModelContextData(newData); + graphCode += "\n###Pure\n"; + } + graphCode += "function " + REPL_RUN_FUNCTION_SIGNATURE + "{\n"; + graphCode += DataCubeHelpers.getQueryCode(input.baseQuery.body.get(0), false) + "\n"; int lineOffset = StringUtils.countMatches(graphCode, "\n"); graphCode += input.code; graphCode += "\n}"; @@ -254,13 +255,12 @@ public HttpHandler getHandler(REPLServerState state) try { DataCubeQuery query = state.getQuery(); + Map source = state.getSource(); if (query != null) { DataCubeGetBaseQueryResult result = new DataCubeGetBaseQueryResult(); - result.timestamp = state.startTime; result.query = query; - result.partialQuery = DataCubeHelpers.parseQuery(query.partialQuery, false); - result.sourceQuery = DataCubeHelpers.parseQuery(query.source.query, false); + result.source = source; handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state); } else diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetBaseQueryResult.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetBaseQueryResult.java index ae339c4e7b8..b73ab865282 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetBaseQueryResult.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetBaseQueryResult.java @@ -14,12 +14,10 @@ package org.finos.legend.engine.repl.dataCube.server.model; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import java.util.Map; public class DataCubeGetBaseQueryResult { public DataCubeQuery query; - public Long timestamp; - public ValueSpecification partialQuery; - public ValueSpecification sourceQuery; + public Map source; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeRelationReturnTypeInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeRelationReturnTypeInput.java index 89953170266..0128dc37020 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeRelationReturnTypeInput.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeRelationReturnTypeInput.java @@ -14,10 +14,11 @@ package org.finos.legend.engine.repl.dataCube.server.model; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; public class DataCubeGetQueryCodeRelationReturnTypeInput { public String code; - public ValueSpecification baseQuery; + public Lambda baseQuery; + public Boolean isolated = false; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryRelationReturnTypeInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryRelationReturnTypeInput.java index 0a8cf232538..f11209f1654 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryRelationReturnTypeInput.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryRelationReturnTypeInput.java @@ -19,4 +19,5 @@ public class DataCubeGetQueryRelationReturnTypeInput { public Lambda query; + public Boolean isolated = false; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeBatchInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetValueSpecificationCodeBatchInput.java similarity index 88% rename from legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeBatchInput.java rename to legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetValueSpecificationCodeBatchInput.java index d96747e07a4..b8098cabc61 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeBatchInput.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetValueSpecificationCodeBatchInput.java @@ -18,8 +18,8 @@ import java.util.Map; -public class DataCubeGetQueryCodeBatchInput +public class DataCubeGetValueSpecificationCodeBatchInput { - public Map queries; + public Map values; public Boolean pretty; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetValueSpecificationCodeInput.java similarity index 89% rename from legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeInput.java rename to legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetValueSpecificationCodeInput.java index acd62d1ad38..7604ba115f3 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetQueryCodeInput.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetValueSpecificationCodeInput.java @@ -16,8 +16,8 @@ import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; -public class DataCubeGetQueryCodeInput +public class DataCubeGetValueSpecificationCodeInput { - public ValueSpecification query; + public ValueSpecification value; public Boolean pretty; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuery.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuery.java index bba8ec8481e..ec6ba099076 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuery.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuery.java @@ -14,18 +14,10 @@ package org.finos.legend.engine.repl.dataCube.server.model; -import com.fasterxml.jackson.annotation.JsonRawValue; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.Map; public class DataCubeQuery { - public String name; public String query; - public String partialQuery; - public DataCubeQuerySource source; - - // NOTE: we don't need to process the config, so we will leave it as raw JSON - @JsonRawValue - @JsonDeserialize(using = DataCubeQueryConfigurationDeserializer.class) - public String configuration; + public Map configuration; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQueryConfigurationDeserializer.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQueryConfigurationDeserializer.java deleted file mode 100644 index 2bcbe962a39..00000000000 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQueryConfigurationDeserializer.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.engine.repl.dataCube.server.model; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.TreeNode; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; - -import java.io.IOException; - -public class DataCubeQueryConfigurationDeserializer extends JsonDeserializer -{ - @Override - public String deserialize(JsonParser jp, DeserializationContext ctx) throws IOException - { - TreeNode tree = jp.getCodec().readTree(jp); - return tree.toString(); - } -} \ No newline at end of file diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuerySource.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuerySource.java deleted file mode 100644 index 710e9f6fac7..00000000000 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuerySource.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.engine.repl.dataCube.server.model; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -import java.util.List; - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type") -@JsonSubTypes({ - @JsonSubTypes.Type(value = DataCubeQuerySourceREPLExecutedQuery.class, name = "REPLExecutedQuery") -}) -public abstract class DataCubeQuerySource -{ - public String query; - public String mapping; - public String runtime; - public List columns; -} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQueryTypeaheadInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQueryTypeaheadInput.java index 84622e749b8..6aabda9618b 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQueryTypeaheadInput.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQueryTypeaheadInput.java @@ -14,10 +14,11 @@ package org.finos.legend.engine.repl.dataCube.server.model; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; public class DataCubeQueryTypeaheadInput { public String code; - public ValueSpecification baseQuery; + public Lambda baseQuery; + public Boolean isolated = false; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/test/java/org/finos/legend/engine/repl/dataCube/TestDataCubeHelpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/test/java/org/finos/legend/engine/repl/dataCube/TestDataCubeHelpers.java index 0e882e37bf6..bc75b1db240 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/test/java/org/finos/legend/engine/repl/dataCube/TestDataCubeHelpers.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/test/java/org/finos/legend/engine/repl/dataCube/TestDataCubeHelpers.java @@ -197,7 +197,15 @@ public void testTypeaheadPartial() { String code = "->extend(~[newCol:c|'ok', colX: c|$c."; String expectedResult = "{\"completion\":[{\"completion\":\"FIRSTNAME\",\"display\":\"FIRSTNAME\"}]}"; - testTypeahead(expectedResult, code, "#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->select(~FIRSTNAME)->from(test::test)"); + testTypeahead(expectedResult, code, (Lambda) DataCubeHelpers.parseQuery("|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->select(~FIRSTNAME)->from(test::test)", false), pureModelContextData); + } + + @Test + public void testTypeaheadPartialWithDummySource() + { + String code = "->extend(~[newCol:c|'ok', colX: c|$c."; + String expectedResult = "{\"completion\":[{\"completion\":\"FIRSTNAME\",\"display\":\"FIRSTNAME\"}]}"; + testTypeahead(expectedResult, code, (Lambda) DataCubeHelpers.parseQuery("|''->cast(@meta::pure::metamodel::relation::Relation<(FIRSTNAME:String)>)", false), null); } @Test @@ -205,7 +213,7 @@ public void testTypeaheadFull() { String code = "#>{test::TestDatabase.TEST0}#->extend(~[newCol:c|'ok', colX: c|$c."; String expectedResult = "{\"completion\":[{\"completion\":\"FIRSTNAME\",\"display\":\"FIRSTNAME\"},{\"completion\":\"LASTNAME\",\"display\":\"LASTNAME\"}]}"; - testTypeahead(expectedResult, code, null); + testTypeahead(expectedResult, code, null, pureModelContextData); } @Test @@ -213,14 +221,14 @@ public void testTypeaheadFullWithError() { String code = "#>{test::TestDatabase.TEST0}#-->extend(~[newCol:c|'ok', colX: c|$c."; String expectedResult = "{\"completion\":[]}"; - testTypeahead(expectedResult, code, null); + testTypeahead(expectedResult, code, null, pureModelContextData); } - private void testTypeahead(String expectedResult, String code, String baseQueryCode) + private void testTypeahead(String expectedResult, String code, Lambda lambda, PureModelContextData data) { try { - Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getCodeTypeahead(code, baseQueryCode, pureModelContextData, completerExtensions, legendInterface))); + Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getCodeTypeahead(code, lambda, data, completerExtensions, legendInterface))); } catch (IOException e) { @@ -233,7 +241,7 @@ public void testExtractRelationReturnTypeGroupBy() { String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()])->from(test::test)"; String expectedResult = "{\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"count\",\"type\":\"Integer\"}]}"; - testExtractRelationReturnType(expectedResult, lambda); + testExtractRelationReturnType(expectedResult, lambda, pureModelContextData); } @Test @@ -241,7 +249,7 @@ public void testExtractRelationReturnTypeCast() { String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(test::test)->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()])->cast(@meta::pure::metamodel::relation::Relation<(hai:String,ba:Integer)>)"; String expectedResult = "{\"columns\":[{\"name\":\"hai\",\"type\":\"String\"},{\"name\":\"ba\",\"type\":\"Integer\"}]}"; - testExtractRelationReturnType(expectedResult, lambda); + testExtractRelationReturnType(expectedResult, lambda, pureModelContextData); } @Test @@ -249,7 +257,7 @@ public void testExtractRelationReturnTypeSimpleExtend() { String lambda = "|#>{test::TestDatabase.TEST0}#->extend(~newCol:c|'ok')"; String expectedResult = "{\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"},{\"name\":\"newCol\",\"type\":\"String\"}]}"; - testExtractRelationReturnType(expectedResult, lambda); + testExtractRelationReturnType(expectedResult, lambda, pureModelContextData); } @Test @@ -257,16 +265,23 @@ public void testExtractRelationReturnTypeMultipleExtend() { String lambda = "|#>{test::TestDatabase.TEST0}#->extend(~[newCol:c|'ok', colX: c|$c.FIRSTNAME])"; String expectedResult = "{\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"},{\"name\":\"newCol\",\"type\":\"String\"},{\"name\":\"colX\",\"type\":\"String\"}]}"; - testExtractRelationReturnType(expectedResult, lambda); + testExtractRelationReturnType(expectedResult, lambda, pureModelContextData); } - private void testExtractRelationReturnType(String expectedResult, String code) + @Test + public void testExtractRelationReturnTypeWithDummySource() + { + String lambda = "|''->cast(@meta::pure::metamodel::relation::Relation<(FIRSTNAME:String,LASTNAME:String)>)->extend(~[newCol:c|'ok', colX: c|$c.FIRSTNAME])"; + String expectedResult = "{\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"},{\"name\":\"newCol\",\"type\":\"String\"},{\"name\":\"colX\",\"type\":\"String\"}]}"; + testExtractRelationReturnType(expectedResult, lambda, null); + } + + private void testExtractRelationReturnType(String expectedResult, String code, PureModelContextData data) { try { Lambda lambda = (Lambda) DataCubeHelpers.parseQuery(code, false); - PureModelContextData data = DataCubeHelpers.injectNewFunction(pureModelContextData, lambda).getOne(); - Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getRelationReturnType(legendInterface, data))); + Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getRelationReturnType(legendInterface, lambda, data))); } catch (IOException e) { @@ -278,23 +293,22 @@ private void testExtractRelationReturnType(String expectedResult, String code) public void testExtractRelationReturnTypeWithParserError() { String lambda = "|#>{test::TestDatabase.TEST0}#-->extend(~[newCol:c|'ok', colX: c|$c.FIRSTNAME])"; - testExtractRelationReturnTypeFailure("PARSER error at [1:31]: Unexpected token '-'. Valid alternatives: [';']", lambda); + testExtractRelationReturnTypeFailure("PARSER error at [1:31]: Unexpected token '-'. Valid alternatives: [';']", lambda, pureModelContextData); } @Test public void testExtractRelationReturnTypeWithCompilationError() { String lambda = "|#>{test::TestDatabase.TEST0}#->extend(~[newCol:c|'ok', colX: c|$c.FIRSTNAME2])"; - testExtractRelationReturnTypeFailure("COMPILATION error at [1:68-77]: The column 'FIRSTNAME2' can't be found in the relation (FIRSTNAME:String, LASTNAME:String)", lambda); + testExtractRelationReturnTypeFailure("COMPILATION error at [1:68-77]: The column 'FIRSTNAME2' can't be found in the relation (FIRSTNAME:String, LASTNAME:String)", lambda, pureModelContextData); } - private void testExtractRelationReturnTypeFailure(String errorMessage, String code) + private void testExtractRelationReturnTypeFailure(String errorMessage, String code, PureModelContextData data) { EngineException e = Assert.assertThrows(EngineException.class, () -> { Lambda lambda = (Lambda) DataCubeHelpers.parseQuery(code, true); - PureModelContextData data = DataCubeHelpers.injectNewFunction(pureModelContextData, lambda).getOne(); - DataCubeHelpers.getRelationReturnType(legendInterface, data); + DataCubeHelpers.getRelationReturnType(legendInterface, lambda, data); }); Assert.assertEquals(errorMessage, EngineException.buildPrettyErrorMessage(e.getMessage(), e.getSourceInformation(), e.getErrorType())); } diff --git a/pom.xml b/pom.xml index 57233ee132a..c87d09647f5 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ 5.20.0 0.25.6 - 12.46.0 + 12.51.0 legend-engine