From 2ef9d8be47fbd603b2f412d846a31766f085b4c1 Mon Sep 17 00:00:00 2001 From: Rafael Bey <24432403+rafaelbey@users.noreply.github.com> Date: Tue, 28 Nov 2023 16:37:17 -0500 Subject: [PATCH] ES support for distinct and restrict functions (#2474) --- .../src/main/resources/core/pure/tds/tds.pure | 5 +- .../legend-engine-pure-ide-light/pom.xml | 7 ++ .../finos/legend/engine/ide/PureIDELight.java | 1 + .../functions/pure_to_elasticsearch.pure | 88 ++++++++++++++----- .../test/shared/ElasticsearchCommands.java | 3 + .../elasticsearch_plan_test_aggregation.pure | 24 +++++ .../elasticsearch_plan_test_project.pure | 30 +++---- ...asticsearch_plan_test_project_boolean.pure | 8 ++ .../elasticsearch_plan_test_project_date.pure | 8 ++ ...elasticsearch_plan_test_project_float.pure | 8 ++ ...asticsearch_plan_test_project_integer.pure | 8 ++ ...asticsearch_plan_test_project_keyword.pure | 9 ++ .../elasticsearch_plan_test_project_text.pure | 9 ++ 13 files changed, 169 insertions(+), 39 deletions(-) diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure index 17bc0e98f5c..b316194352e 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure @@ -453,8 +453,9 @@ function {doc.doc = 'Remove duplicate rows from the priovded TDS'} meta::pure::tds::distinct(tds:TabularDataSet[1]):TabularDataSet[1] { - fail('Not implemented yet!'); - $tds; + let newTds = ^$tds(rows = []); + let distinctRows = $tds.rows->removeDuplicates({l, r | $l.values == $r.values})->map(r | ^$r(parent = $newTds)); + $newTds->mutateAdd('rows', $distinctRows); } function diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/pom.xml b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/pom.xml index 7089c3ab7fa..f505e3c098d 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/pom.xml +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/pom.xml @@ -121,6 +121,13 @@ + + org.finos.legend.engine + legend-engine-xt-elasticsearch-executionPlan-test + runtime + ${project.version} + + com.smoketurner dropwizard-swagger diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java index 5a20a181b37..185551de86c 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java @@ -100,6 +100,7 @@ protected MutableList buildRepositories(SourceLocationCon .with(this.buildCore("legend-engine-xts-sql/legend-engine-xt-sql-pure", "external-query-sql")) .with(this.buildCore("legend-engine-xts-authentication/legend-engine-xt-authentication-pure", "authentication")) .with(this.buildCore("legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-pure-specification-metamodel", "elasticsearch_specification_metamodel")) + .with(this.buildCore("legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test", "elasticsearch_execution_test")) .with(this.buildCore("legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel", "elasticsearch_seven_metamodel")) .with(this.buildCore("legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-pure","nonrelational-mongodb")) .with(this.buildCore("legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-javaPlatformBinding-pure","nonrelational-mongodb-java-platform-binding")) diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure index 6c9344f042b..28c81d44034 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure @@ -81,7 +81,6 @@ Class meta::external::store::elasticsearch::v7::pureToEs::TDSESDetail path(){ $this.resultPath.path() }:String[1]; - expression: FunctionExpression[0..1]; format: String[0..1]; } @@ -124,9 +123,9 @@ function meta::external::store::elasticsearch::v7::pureToEs::resultPathToQuery(t let srps = $tdsDetails.resultPath->concatenate($frps.readFrom)->filter(x | $x->instanceOf(SourceFieldResultPath))->cast(@SourceFieldResultPath); let docps = $tdsDetails.resultPath->concatenate($frps.readFrom)->filter(x | $x->instanceOf(DocValueResultPath))->cast(@DocValueResultPath); - let seachWithSrps = $srps->isNotEmpty()->if(| ^$search(_source = ^SourceConfig(filter = ^SourceFilter(includes = $srps.path()->literal()))), | $search); + let searchWithSrps = $srps->isNotEmpty()->if(| ^$search(_source = ^SourceConfig(filter = ^SourceFilter(includes = $srps.path()->literal()))), | $search); let fields = $docps->map(x | $x->resultPathToFieldAndFormat($tdsDetails->filter(t | $t.resultPath == $x)->first())); - $fields->isEmpty()->if(|$seachWithSrps, |^$seachWithSrps(docvalue_fields = $fields)); + $fields->isEmpty()->if(|$searchWithSrps, |^$searchWithSrps(docvalue_fields = $fields)); } function meta::external::store::elasticsearch::v7::pureToEs::resultPathToFieldAndFormat(resultPath: DocValueResultPath[1], tdsEsDetail: TDSESDetail[0..1]): FieldAndFormat[1] @@ -135,6 +134,29 @@ function meta::external::store::elasticsearch::v7::pureToEs::resultPathToFieldAn ^FieldAndFormat(field = $resultPath.path()->literal(), format = $format->literal()); } +function meta::external::store::elasticsearch::v7::pureToEs::processRestrict(vs : FunctionExpression[1], initReq: State[1]): State[1] +{ + $initReq.debug(|'Processing ->restrict'); + let currReq = process($vs.parametersValues->at(0), $initReq); + let projectColNames = $vs->instanceValuesAtParameter(1, $currReq.sq.inScopeVars)->cast(@String); + let toRestrict = $projectColNames->map(s | $currReq.tdsESDetails->filter(x | $x.name == $s)); + assert($toRestrict->size() == $projectColNames->size(), | 'restricting by unknown columns: ' + $projectColNames->removeAll($toRestrict.name)->joinStrings('[', ', ', ']')); + + let search = $currReq.search; + ^$currReq( + search = resultPathToQuery($toRestrict, $search), + tdsESDetails = $toRestrict + ); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processDistinct(vs : FunctionExpression[1], initReq: State[1]): State[1] +{ + $initReq.debug(|'Processing ->distinct'); + let currReq = process($vs.parametersValues->at(0), $initReq); + assertFalse($currReq.aggregationQuery, |'distinct not supported in aggregation queries'); + processGroupBy($currReq.tdsESDetails, [], $currReq); +} + function meta::external::store::elasticsearch::v7::pureToEs::processExtend(vs : FunctionExpression[1], initReq: State[1]): State[1] { $initReq.debug(|'Processing ->extend'); @@ -151,7 +173,7 @@ function meta::external::store::elasticsearch::v7::pureToEs::processProject(vs : { let currReq = process($vs.parametersValues->at(0), $initReq); assert(!$currReq.inFilter); - assertFalse($currReq.aggregationQuery, |'project not supported in aggreagtion queries'); + assertFalse($currReq.aggregationQuery, |'project not supported in aggregation queries'); let cols = $vs->instanceValuesAtParameter(1, $currReq.sq.inScopeVars); let fieldsStatePair = $cols->match([ tdsCols: BasicColumnSpecification[*] | $tdsCols->fold( @@ -180,12 +202,22 @@ function meta::external::store::elasticsearch::v7::pureToEs::processProjectColum $expr->extractSimpleValue($initReq).first.values->match([ tdsDetail: TDSESDetail[1] | pair(^$tdsDetail(name = $name, type = $type), $initReq), - {fe: FunctionExpression[1] | + {vs: ValueSpecification[1] | let resultPath = ^DocValueResultPath(fieldPath = $name, property = $type->defaultRuntimePropertyForPureType()); - let scripted = $fe->toRuntimeMapping($resultPath, $initReq); - pair(^TDSESDetail(type = $type, name = $name, resultPath = $resultPath, expression = $fe, format = $scripted.search.runtime_mappings->toOne()->get($name).format.value), $scripted); + let scripted = $vs->toRuntimeMapping($resultPath, $initReq); + pair(^TDSESDetail(type = $type, name = $name, resultPath = $resultPath, format = $scripted.search.runtime_mappings->toOne()->get($name).format.value), $scripted); }, - any: Any[*] | fail(|'Cannot project column - %s:%s'->format([$name,$any->type()->elementToPath()]))->cast(@Pair) + {any: Any[*] | + $any->type()->match([ + {pt: PrimitiveType[1] | + let resultPath = ^DocValueResultPath(fieldPath = $name, property = $pt->defaultRuntimePropertyForPureType()); + let iv = ^InstanceValue(multiplicity = $vs.func->functionReturnMultiplicity(), genericType = ^GenericType(rawType = $type), values=$any)->evaluateAndDeactivate(); + let scripted = $iv->toRuntimeMapping($resultPath, $initReq); + pair(^TDSESDetail(type = $type, name = $name, resultPath = $resultPath, format = $scripted.search.runtime_mappings->toOne()->get($name).format.value), $scripted); + }, + other: Any[*] | fail(|'Cannot project column - %s:%s'->format([$name,$any->type()->elementToPath()]))->cast(@Pair) + ]) + } ]); } @@ -273,6 +305,11 @@ function meta::external::store::elasticsearch::v7::pureToEs::processGroupBy(vs: let groupByTdsESDetails = $groupByCols->map(g | $groupedReq.tdsESDetails->filter(x | $x.name == $g)); assert($groupByCols->size() == $groupByTdsESDetails->size(), | 'grouping by unknown columns: ' + $groupByCols->removeAll($groupByTdsESDetails.name)->joinStrings('[', ', ', ']')); + processGroupBy($groupByTdsESDetails, $aggregateValues, $groupedReq); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processGroupBy(groupByTdsESDetails: TDSESDetail[*], aggregateValues: meta::pure::tds::AggregateValue[*], groupedReq: State[1]): State[1] +{ let aggPairs = $aggregateValues->map({x | let rawToAggregate = processProjectColumn(^BasicColumnSpecification(func = $x.mapFn, name = $x.name), $groupedReq).first; let aggFunc = $x.aggregateFn->deepByPassRouterInfo()->cast(@FunctionDefinition).expressionSequence->toOne('tds aggregation only supports simple expressions: max, min, sum, etc.'); @@ -293,7 +330,7 @@ function meta::external::store::elasticsearch::v7::pureToEs::processGroupBy(vs: }); let aggregations = newMap($aggPairs->map(x | pair($x.first.name, $x.second))); - let aggregationsForSearch = if ($groupByCols->isEmpty(), + let aggregationsForSearch = if ($groupByTdsESDetails->isEmpty(), | // no group by field - just run the aggregations $aggregations @@ -316,6 +353,7 @@ function meta::external::store::elasticsearch::v7::pureToEs::processGroupBy(vs: let newSearch = ^$search( size = 0->literal() // avoiding reading all the matches, we just want the aggregate results ,aggregations = $aggregationsForSearch + ,docvalue_fields = [] ,_source = ^SourceConfig(fetch = false->literal()) ); @@ -558,22 +596,24 @@ function meta::external::store::elasticsearch::v7::pureToEs::supportedRoutingFun { let supported = [ pair(supportedIfEqual(indexToTDS_Elasticsearch7Store_1__String_1__TabularDataSet_1_), processSelectAllTds_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::project_TabularDataSet_1__ColumnSpecification_MANY__TabularDataSet_1_), processProject_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::extend_TabularDataSet_1__BasicColumnSpecification_MANY__TabularDataSet_1_), processExtend_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(project_TabularDataSet_1__ColumnSpecification_MANY__TabularDataSet_1_), processProject_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(extend_TabularDataSet_1__BasicColumnSpecification_MANY__TabularDataSet_1_), processExtend_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(restrict_TabularDataSet_1__String_MANY__TabularDataSet_1_), processRestrict_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(distinct_TabularDataSet_1__TabularDataSet_1_), processDistinct_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::filter_TabularDataSet_1__Function_1__TabularDataSet_1_), processFilter_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::sort_TabularDataSet_1__String_MANY__TabularDataSet_1_), processDefaultSort_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::sort_TabularDataSet_1__SortInformation_MANY__TabularDataSet_1_), processSortWithInformation_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::sort_TabularDataSet_1__String_1__SortDirection_1__TabularDataSet_1_), processSortWithDirection_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(filter_TabularDataSet_1__Function_1__TabularDataSet_1_), processFilter_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::groupBy_TabularDataSet_1__String_MANY__AggregateValue_MANY__TabularDataSet_1_), processGroupBy_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(sort_TabularDataSet_1__String_MANY__TabularDataSet_1_), processDefaultSort_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(sort_TabularDataSet_1__SortInformation_MANY__TabularDataSet_1_), processSortWithInformation_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(sort_TabularDataSet_1__String_1__SortDirection_1__TabularDataSet_1_), processSortWithDirection_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(groupBy_TabularDataSet_1__String_MANY__AggregateValue_MANY__TabularDataSet_1_), processGroupBy_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::limit_TabularDataSet_1__Integer_1__TabularDataSet_1_), processLimit_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::take_TabularDataSet_1__Integer_1__TabularDataSet_1_), processLimit_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::drop_TabularDataSet_1__Integer_1__TabularDataSet_1_), processDrop_FunctionExpression_1__State_1__State_1_), - pair(supportedIfEqual(meta::pure::tds::slice_TabularDataSet_1__Integer_1__Integer_1__TabularDataSet_1_), processSlice_FunctionExpression_1__State_1__State_1_) + pair(supportedIfEqual(limit_TabularDataSet_1__Integer_1__TabularDataSet_1_), processLimit_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(take_TabularDataSet_1__Integer_1__TabularDataSet_1_), processLimit_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(drop_TabularDataSet_1__Integer_1__TabularDataSet_1_), processDrop_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(slice_TabularDataSet_1__Integer_1__Integer_1__TabularDataSet_1_), processSlice_FunctionExpression_1__State_1__State_1_) ]; } @@ -1228,7 +1268,7 @@ function meta::external::store::elasticsearch::v7::pureToEs::varFreemarkerExpres )); } -function meta::external::store::elasticsearch::v7::pureToEs::toRuntimeMapping(vs: FunctionExpression[1], field: DocValueResultPath[1], initReq: State[1]): State[1] +function meta::external::store::elasticsearch::v7::pureToEs::toRuntimeMapping(vs: ValueSpecification[1], field: DocValueResultPath[1], initReq: State[1]): State[1] { let p = processPainless($vs, ^$initReq(inProject = false, inFilter = true)); @@ -1247,7 +1287,10 @@ function meta::external::store::elasticsearch::v7::pureToEs::toRuntimeMapping(vs let emittingScript = ^$script(source = 'emit(%s%s)'->format([$script.source.value->toOne(), $toEpochMillis])->literal()); - let format = painlessDateFunctionsFormat($vs.func)->literal(); + let format = $vs->match([ + fe: FunctionExpression[1] | painlessDateFunctionsFormat($fe.func), + any: Any[*] | [] + ])->literal(); let newField = pair($field.path(), ^RuntimeField(script = ^Script(inline = $emittingScript), format = $format, type = $type)); @@ -1267,6 +1310,7 @@ function meta::external::store::elasticsearch::v7::pureToEs::processPainless(vs: { $vs->match([ fe: FunctionExpression[1] | $initReq.supportedForPainlessScriptFunctions->findAndEvalSupportedFunction($fe, $initReq, $initReq), + ve: VariableExpression[1] | $ve->processPainlessScalarValue($initReq), iv: InstanceValue[1] | $iv.values->match([ f: FunctionDefinition[1] | $f.expressionSequence->at(0)->processPainless($initReq), diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/shared/ElasticsearchCommands.java b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/shared/ElasticsearchCommands.java index 893eb6fa3bf..b7b80238035 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/shared/ElasticsearchCommands.java +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/shared/ElasticsearchCommands.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.UUID; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; @@ -52,6 +53,7 @@ public class ElasticsearchCommands public static Root_meta_pure_functions_io_http_URL startServer(String imageTag) { + System.setProperty("org.finos.legend.engine.plan.execution.stores.elasticsearch.test.password", UUID.randomUUID().toString()); Root_meta_pure_functions_io_http_URL_Impl url = new Root_meta_pure_functions_io_http_URL_Impl("esUrl"); ElasticsearchContainer container = CONTAINERS.computeIfAbsent(imageTag, ElasticsearchCommands::createContainer); url._host(container.getHost()); @@ -64,6 +66,7 @@ public static Root_meta_pure_functions_io_http_URL startServer(String imageTag) public static void stopServer(String imageTag) { + System.clearProperty("org.finos.legend.engine.plan.execution.stores.elasticsearch.test.password"); Optional.ofNullable(CONTAINERS.remove(imageTag)).ifPresent(ElasticsearchContainer::stop); } diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_aggregation.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_aggregation.pure index 0c82c6b7dcb..c0b96ec649e 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_aggregation.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_aggregation.pure @@ -65,4 +65,28 @@ function meta::external::store::elasticsearch::executionTest::testCase::tds::groupBy::testGroupByOnNullableField(config:TestConfig[1]):Boolean[1] { $config->testTdsExpression(x|$x->groupBy('MPAA', agg('count', r | $r.getNullableString('MPAA'), agg | $agg->count()))); +} + +function + <> + {doc.doc = 'Test distinct on single columns on Elasticsearch (translated as a group by)'} +meta::external::store::elasticsearch::executionTest::testCase::tds::distinct::testDistinctSingleColumn(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->restrict('Director')->distinct()); +} + +function + <> + {doc.doc = 'Test distinct on multiple columns on Elasticsearch (translated as a group by)'} +meta::external::store::elasticsearch::executionTest::testCase::tds::distinct::testDistinctMultipleColumns(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->restrict(['Director', 'MPAA'])->distinct()); +} + +function + <> + {doc.doc = 'Test restrict group by columns on Elasticsearch'} +meta::external::store::elasticsearch::executionTest::testCase::tds::restrict::testRestrictGroupByColumns(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->groupBy(['Director', 'MPAA'], [ agg('sumBudget', r | $r.getInteger('Budget'), agg | $agg->sum()), agg('avgBudget', r | $r.getInteger('Budget'), agg | $agg->average()) ])->restrict(['Director', 'avgBudget'])); } \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure index 201f78cf3cd..0bb015433d2 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure @@ -18,22 +18,22 @@ import meta::external::store::elasticsearch::executionTest::testCase::tds::*; import meta::external::store::elasticsearch::executionTest::test::*; import meta::external::store::elasticsearch::executionTest::utils::*; -// function -// <> -// {doc.doc = 'Test projection on Elasticsearch with pure constant expressions'} -// meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionConstant(config:TestConfig[1]):Boolean[1] -// { -// $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | 'Hello World!', 'Bucket')])); -// } +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure constant expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionConstant(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | 'Hello World!', 'Bucket')])); +} -// function -// <> -// {doc.doc = 'Test projection on Elasticsearch with pure constant expressions'} -// meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionConstantWithVariable(config:TestConfig[1]):Boolean[1] -// { -// let val = 12345; -// $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $val, 'Bucket')])); -// } +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure constant expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionConstantWithVariable(config:TestConfig[1]):Boolean[1] +{ + let val = 12345; + $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $val, 'Bucket')])); +} function <> diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_boolean.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_boolean.pure index f2ec3195bd2..3003b8f7e23 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_boolean.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_boolean.pure @@ -18,6 +18,14 @@ import meta::external::store::elasticsearch::executionTest::testCase::tds::*; import meta::external::store::elasticsearch::executionTest::test::*; import meta::external::store::elasticsearch::executionTest::utils::*; +function + <> + {doc.doc = 'Test restrict on Elasticsearch on boolean field'} +meta::external::store::elasticsearch::executionTest::testCase::tds::restrict::boolean::testRestrictBoolean(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | $x.isNull('MPAA'), 'has mpaa')])->restrict('has mpaa')); +} + function <> {doc.doc = 'Test projection on Elasticsearch with pure isEmpty expressions'} diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_date.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_date.pure index d14eb85760e..1e80b3cd75e 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_date.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_date.pure @@ -18,6 +18,14 @@ import meta::external::store::elasticsearch::executionTest::testCase::tds::*; import meta::external::store::elasticsearch::executionTest::test::*; import meta::external::store::elasticsearch::executionTest::utils::*; +function + <> + {doc.doc = 'Test restrict on Elasticsearch Date property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::restrict::date::testRestrictDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->restrict('ReleaseDate')); +} + function <> {doc.doc = 'Test projection on Elasticsearch Date property mapping'} diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_float.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_float.pure index 34563842ef7..41c389aa966 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_float.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_float.pure @@ -18,6 +18,14 @@ import meta::external::store::elasticsearch::executionTest::testCase::tds::*; import meta::external::store::elasticsearch::executionTest::test::*; import meta::external::store::elasticsearch::executionTest::utils::*; +function + <> + {doc.doc = 'Test restrict on Elasticsearch Float property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::restrict::float::testRestrictFloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->restrict('Revenue')); +} + function <> {doc.doc = 'Test projection on Elasticsearch Float property mapping'} diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_integer.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_integer.pure index 058c6dc38fe..48912f15d32 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_integer.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_integer.pure @@ -18,6 +18,14 @@ import meta::external::store::elasticsearch::executionTest::testCase::tds::*; import meta::external::store::elasticsearch::executionTest::test::*; import meta::external::store::elasticsearch::executionTest::utils::*; +function + <> + {doc.doc = 'Test restrict on Elasticsearch Integer property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::restrict::integer::testRestrictInteger(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->restrict('Budget')); +} + function <> {doc.doc = 'Test projection on Elasticsearch Integer property mapping'} diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_keyword.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_keyword.pure index 16ef0069ea8..749ebed6458 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_keyword.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_keyword.pure @@ -18,6 +18,15 @@ import meta::external::store::elasticsearch::executionTest::testCase::tds::*; import meta::external::store::elasticsearch::executionTest::test::*; import meta::external::store::elasticsearch::executionTest::utils::*; +function + <> + {doc.doc = 'Test restrict on Elasticsearch Keyword property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::restrict::keyword::testRestrictKeyword(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->restrict('Title')); +} + + function <> {doc.doc = 'Test projection on Elasticsearch Keyword property mapping'} diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_text.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_text.pure index 2bd4aa98876..536e04a99e8 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_text.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_text.pure @@ -18,6 +18,15 @@ import meta::external::store::elasticsearch::executionTest::testCase::tds::*; import meta::external::store::elasticsearch::executionTest::test::*; import meta::external::store::elasticsearch::executionTest::utils::*; +function + <> + {doc.doc = 'Test restrict on Elasticsearch Text property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::restrict::text::testRestrictText(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->restrict('Description')); +} + + function <> {doc.doc = 'Test projection on Elasticsearch Text property mapping'}