From 3f04b21b2bbdd10cf26d82e1611a3d957b65e208 Mon Sep 17 00:00:00 2001 From: Rafael Bey <24432403+rafaelbey@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:04:20 -0400 Subject: [PATCH] Elasticsearch - support for complex expression on TDS project/extend (#2356) --- .../legend-engine-server/pom.xml | 26 +- .../finos/legend/engine/server/Server.java | 8 +- .../TDSColumnWithSerializer.java | 18 +- .../pure/router/routing/router_routing.pure | 66 +- .../src/main/resources/core/pure/tds/tds.pure | 21 +- .../extensions/store_contract.pure | 2 +- .../functions/pure_to_elasticsearch.pure | 725 ++++++++++++++---- .../functions/specification_utils.pure | 20 + .../test/shared/ElasticsearchCommands.java | 5 +- .../elasticsearch_plan_test.pure | 38 +- .../elasticsearch_plan_test_7.pure | 22 +- ...lasticsearch_plan_test_filter_boolean.pure | 39 + .../elasticsearch_plan_test_filter_date.pure | 30 +- .../elasticsearch_plan_test_filter_enum.pure | 31 + ...icsearch_plan_test_filter_expressions.pure | 6 +- .../elasticsearch_plan_test_misc.pure | 72 +- .../elasticsearch_plan_test_project.pure | 60 +- ...asticsearch_plan_test_project_boolean.pure | 87 +++ .../elasticsearch_plan_test_project_date.pure | 238 ++++++ ...elasticsearch_plan_test_project_float.pure | 198 +++++ ...asticsearch_plan_test_project_integer.pure | 190 +++++ ...asticsearch_plan_test_project_keyword.pure | 61 ++ .../elasticsearch_plan_test_project_text.pure | 77 ++ ...chExecutionPlanFromGrammarIntegration.java | 2 +- 24 files changed, 1747 insertions(+), 295 deletions(-) create mode 100644 legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_boolean.pure create mode 100644 legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_enum.pure create mode 100644 legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_boolean.pure create mode 100644 legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_date.pure create mode 100644 legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_float.pure create mode 100644 legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_integer.pure create mode 100644 legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_keyword.pure create mode 100644 legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_text.pure diff --git a/legend-engine-config/legend-engine-server/pom.xml b/legend-engine-config/legend-engine-server/pom.xml index 02789e87e86..532d877188a 100644 --- a/legend-engine-config/legend-engine-server/pom.xml +++ b/legend-engine-config/legend-engine-server/pom.xml @@ -672,6 +672,27 @@ + + org.finos.legend.engine + legend-engine-xt-elasticsearch-V7-executionPlan + ${project.version} + + + org.finos.legend.engine + legend-engine-extensions-collection-generation + runtime + + + org.finos.legend.engine + legend-engine-extensions-collection-execution + runtime + + + org.bouncycastle + * + + + org.finos.legend.engine legend-engine-xt-authentication-connection-factory @@ -689,6 +710,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-snowflake-connection + @@ -912,12 +934,12 @@ - junit junit - test + runtime + diff --git a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java index ad248e0062b..45ea6d8881a 100644 --- a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java +++ b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java @@ -102,6 +102,8 @@ import org.finos.legend.engine.plan.execution.concurrent.ParallelGraphFetchExecutionExecutorPool; import org.finos.legend.engine.plan.execution.graphFetch.GraphFetchExecutionConfiguration; import org.finos.legend.engine.plan.execution.service.api.ServiceModelingApi; +import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutor; +import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutorBuilder; import org.finos.legend.engine.plan.execution.stores.inMemory.plugin.InMemory; import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutor; import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutorBuilder; @@ -292,12 +294,14 @@ public void run(T serverConfiguration, Environment environment) MongoDBStoreExecutorConfiguration mongoDBExecutorConfiguration = MongoDBStoreExecutorConfiguration.newInstance().withCredentialProviderProvider(credentialProviderProvider).build(); MongoDBStoreExecutor mongoDBStoreExecutor = (MongoDBStoreExecutor) new MongoDBStoreExecutorBuilder().build(mongoDBExecutorConfiguration); + ElasticsearchV7StoreExecutor elasticsearchV7StoreExecutor = (ElasticsearchV7StoreExecutor) new ElasticsearchV7StoreExecutorBuilder().build(); + PlanExecutor planExecutor; ParallelGraphFetchExecutionExecutorPool parallelGraphFetchExecutionExecutorPool = null; if (serverConfiguration.graphFetchExecutionConfiguration != null) { GraphFetchExecutionConfiguration graphFetchExecutionConfiguration = serverConfiguration.graphFetchExecutionConfiguration; - planExecutor = PlanExecutor.newPlanExecutor(graphFetchExecutionConfiguration, relationalStoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); + planExecutor = PlanExecutor.newPlanExecutor(graphFetchExecutionConfiguration, relationalStoreExecutor, elasticsearchV7StoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); if (graphFetchExecutionConfiguration.canExecuteInParallel()) { parallelGraphFetchExecutionExecutorPool = new ParallelGraphFetchExecutionExecutorPool(graphFetchExecutionConfiguration.getParallelGraphFetchExecutionConfig(), "thread-pool for parallel graphFetch execution"); @@ -306,7 +310,7 @@ public void run(T serverConfiguration, Environment environment) } else { - planExecutor = PlanExecutor.newPlanExecutor(relationalStoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); + planExecutor = PlanExecutor.newPlanExecutor(relationalStoreExecutor, elasticsearchV7StoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); } // Session Management diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/serialization/TDSColumnWithSerializer.java b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/serialization/TDSColumnWithSerializer.java index 2dc2862695d..d70b4e16b46 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/serialization/TDSColumnWithSerializer.java +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/serialization/TDSColumnWithSerializer.java @@ -15,13 +15,12 @@ package org.finos.legend.engine.plan.execution.result.serialization; import com.fasterxml.jackson.core.JsonGenerator; -import org.eclipse.collections.impl.block.procedure.checked.ThrowingProcedure2; -import org.finos.legend.engine.plan.dependencies.domain.date.PureDate; -import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.TDSColumn; - import java.io.IOException; import java.math.BigDecimal; import java.util.Objects; +import org.eclipse.collections.impl.block.procedure.checked.ThrowingProcedure2; +import org.finos.legend.engine.plan.dependencies.domain.date.PureDate; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.TDSColumn; public class TDSColumnWithSerializer { @@ -51,9 +50,16 @@ public TDSColumnWithSerializer(TDSColumn tdsColumn) return (ThrowingProcedure2) JsonGenerator::writeBoolean; case "Date": case "DateTime": - return (ThrowingProcedure2) (jg, d) -> jg.writeString(d.toInstant().toString()); case "StrictDate": - return (ThrowingProcedure2) (jg, d) -> jg.writeString(d.toLocalDate().toString()); + return (ThrowingProcedure2) (jg, d) -> + { + String formatted = d.toString(); + if (d.hasMinute()) + { + formatted += "Z"; + } + jg.writeString(formatted); + }; default: throw new UnsupportedOperationException("TDS type not supported: " + type); } diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure index 816b1e27d55..e21a30fc15d 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure @@ -500,60 +500,19 @@ function meta::pure::router::routing::processColumnSpecification(v:ValueSpecific function meta::pure::router::routing::processColSpecParams(fe:FunctionExpression[1], state:RoutingState[1], executionContext:ExecutionContext[1], vars:Map[1], inScopeVars:Map>[1], extensions:meta::pure::extension::Extension[*], debug:DebugContext[1]):Any[*] { let routedFe = ^$state(value=$fe)->routeValueSpecification($executionContext, $vars, $inScopeVars, $extensions, $debug); - let params = $routedFe.value->evaluateAndDeactivate()->cast(@FunctionExpression)->toOne().parametersValues->map(p|$p->byPassRouterInfo()->match([r:FunctionRoutedValueSpecification[1]|$r.originalFunction;, v:ValueSpecification[1]|$v]))->cast(@ValueSpecification); - let newParams = $params->map(p|if($p->instanceOf(SimpleFunctionExpression),| $p->processColumnSpecification($state, $executionContext, $vars, $inScopeVars, $extensions, $debug).value->cast(@ValueSpecification),| $p)); - let nfe = ^$fe(parametersValues=$newParams); - - // In plan generation flow, not always all variables will be present in inScopeVars as they could be function params (known only at runtime) - // Hence, reactivating with empty lists in such situations (as the expression of lambda is not expected to be reactivated) - $nfe->reactivateWithDummyValuesForOpenVariables($inScopeVars)->evaluateAndDeactivate(); + let parametersValues = $routedFe.value->evaluateAndDeactivate()->cast(@FunctionExpression)->toOne().parametersValues->map(x | $x->byPassRouterInfo()->match([ + i: InstanceValue[1] | $i.values->match([ + // workaround a compile execution behavior that check for open variables and cause expressions to fail reactivation (does not happen on interpreted) + l: LambdaFunction[1] | ^$i(values = ^$l(openVariables = [])), + a: Any[*] | $i + ]), + a: ValueSpecification[*] | $a + ]))->cast(@ValueSpecification); + + let nfe = ^$fe(parametersValues = $parametersValues); + let colSpec = $nfe->reactivate($inScopeVars)->evaluateAndDeactivate()->cast(@ColumnSpecification)->toOne(); } -function meta::pure::router::routing::reactivateWithDummyValuesForOpenVariables(vs:ValueSpecification[1], inScopeVars:Map>[1]) : Any[*] -{ - let lambdas = $vs->extractLambdasFromVS(); - let newInScopeVars = $lambdas->fold( {l, vars | $vars->meta::pure::router::routing::addDummyValuesforOpenVariables($l)}, $inScopeVars) ; - $vs->reactivate($newInScopeVars); -} - -function meta::pure::router::routing::addDummyValuesforOpenVariables( inScopeVars : Map>[1], l :LambdaFunction[1] ) : Map>[1] -{ - $l.openVariables->fold({v, vars| if($vars->get($v)->isEmpty(), | $vars-> meta::pure::functions::collection::put($v, list([])); , | $vars) }, $inScopeVars); -} - -function meta::pure::router::routing::extractLambdasFromAny(a:Any[1]): LambdaFunction[*] -{ - $a->match([ - f:Function[1] | $f->extractLambdasFromFunction();, - vs: ValueSpecification[1] | $vs->extractLambdasFromVS();, - other: Any[1] | [] - ]); -} - -function meta::pure::router::routing::extractLambdasFromVS(vs:ValueSpecification[1]):LambdaFunction[*] -{ - $vs->match([ - e:ExtendedRoutedValueSpecification[1] | $e.value->evaluateAndDeactivate()->extractLambdasFromVS() , - f:FunctionRoutedValueSpecification[1] | $f.value->evaluateAndDeactivate()->extractLambdasFromVS(), - r:NoSetRoutedValueSpecification[1] | $r.value->evaluateAndDeactivate()->extractLambdasFromVS(), - fe:FunctionExpression[1] | $fe.func->evaluateAndDeactivate()->extractLambdasFromFunction()->concatenate($fe.parametersValues->evaluateAndDeactivate()->map(p| $p->extractLambdasFromVS())), - i:InstanceValue[1] | $i.values->evaluateAndDeactivate()->map(v|$v->extractLambdasFromAny()), - va:VariableExpression[1] | [], - vs:ValueSpecification[1] | [] - ]); -} - -function meta::pure::router::routing::extractLambdasFromFunction(f:Function[1]): LambdaFunction[*] -{ - $f->match([ - p:Property[1] | [], - nf:NativeFunction[1] | [], - l: LambdaFunction[1] | $l, - qp:QualifiedProperty[1] | $qp.expressionSequence->evaluateAndDeactivate()->map(vs|$vs->extractLambdasFromVS());, - cfd:ConcreteFunctionDefinition[1] | $cfd.expressionSequence->evaluateAndDeactivate()->map(vs|$vs->extractLambdasFromVS());, - f:Function[1] | [] - ]); -} function meta::pure::router::routing::processAggregationFunctionExpression(aggFuncExpr:FunctionExpression[1], routed:ExtendedRoutedValueSpecification[*], v:RoutingState[1], executionContext:ExecutionContext[1], vars:Map[1], inScopeVars:Map>[1], extensions:meta::pure::extension::Extension[*], debug:DebugContext[1]):RoutingState[1] { @@ -679,6 +638,9 @@ function meta::pure::router::routing::shouldStopFunctions(extensions:meta::pure: groupBy_K_MANY__Function_MANY__AggregateValue_MANY__String_MANY__TabularDataSet_1_, groupBy_TabularDataSet_1__String_MANY__AggregateValue_MANY__TabularDataSet_1_, distinct_TabularDataSet_1__TabularDataSet_1_, + month_Date_1__Month_1_, + quarter_Date_1__Quarter_1_, + dayOfWeek_Date_1__DayOfWeek_1_, mostRecentDayOfWeek_DayOfWeek_1__Date_1_, mostRecentDayOfWeek_Date_1__DayOfWeek_1__Date_1_, previousDayOfWeek_DayOfWeek_1__Date_1_, diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure index 9ef0a2220ab..17bc0e98f5c 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure @@ -441,7 +441,8 @@ function <> meta::pure::tds::multipleColumnComp(row:TDSRow[1], o if ($sortInfo->isEmpty(), | 0, | let s = $sortInfo->head(); - let res = $row.get($s.column->toOne())->compare($other.get($s.column->toOne())); + let res = if($row.isNull($s.column->toOne()), |^TDSNull(), |$row.get($s.column->toOne())) + ->compare(if($other.isNull($s.column->toOne()), |^TDSNull(), |$other.get($s.column->toOne()))); if ($res == 0, |$row->multipleColumnComp($other, $sortInfo->tail()), |if ($s.direction == SortDirection.ASC, |$res,|-$res)); ); } @@ -541,7 +542,15 @@ function //todo: remove this by making parent an association $newTds->mutateAdd('rows', $tds.rows->map(r | ^TDSRow(parent=$newTds, - values=$r.values->concatenate($newColumnFunctions.func->map(f | $f->evaluate(^List(values=[$r]))))))); + values=$r.values->concatenate( + $newColumnFunctions.func->map(f | + let value = $f->evaluate(^List(values=[$r])); + if($value->isEmpty(), |^TDSNull(), |$value); + ) + ) + ) + ) + ); $newTds; } @@ -559,7 +568,13 @@ function //todo: remove this by making parent an association $newTds->mutateAdd('rows', $tds.rows->map(r | ^TDSRow(parent=$newTds, - values=$newColumnFunctions.func->map(f | $f->evaluate(^List(values=[$r])))))); + values=$newColumnFunctions.func->map(f | + let value = $f->evaluate(^List(values=[$r])); + if($value->isEmpty(), |^TDSNull(), |$value); + ) + ) + ) + ); $newTds; } diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/extensions/store_contract.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/extensions/store_contract.pure index 024ac56177c..5f2da2e7f64 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/extensions/store_contract.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/extensions/store_contract.pure @@ -130,7 +130,7 @@ function meta::external::store::elasticsearch::v7::contract::supports(): Functio { let supportedFunctions = meta::external::store::elasticsearch::v7::pureToEs::supportedRoutingFunctions().first; - {f:FunctionExpression[1]| $supportedFunctions->exists(x | $x->eval($f.func))}; + {f:FunctionExpression[1]| $supportedFunctions->exists(x | $x->eval($f.func, []))}; } function meta::external::store::elasticsearch::v7::contract::shouldStopPreeval(): Function<{Any[*]->Boolean[1]}>[1] diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure index e24ba31b711..f5d185d6b19 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import meta::pure::functions::hash::*; import meta::pure::platform::executionPlan::generation::*; import meta::pure::executionPlan::*; import meta::external::store::elasticsearch::v7::specification::utils::*; @@ -43,6 +42,7 @@ import meta::external::store::elasticsearch::v7::metamodel::executionPlan::conte Class meta::external::store::elasticsearch::v7::pureToEs::State { + counter: Integer[1] = 1; search: SearchRequestBody[1]; inFilter: Boolean[1]; inProject: Boolean[1]; @@ -65,10 +65,11 @@ Class meta::external::store::elasticsearch::v7::pureToEs::State tdsESDetails: TDSESDetail[*]; - supportedRoutingFunctions: Pair[1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->State[1]}>>[*]; - supportedFilterFunctions: Pair[1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->State[1]}>>[*]; - supportedSimpleValueFunctions: Pair[1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->Pair, State>[1]}>>[*]; - supportedAggregationFunctions: Pair[1]->Boolean[1]}>, Function<{FunctionExpression[1], String[1]->AggregationContainer[1]}>>[*]; + supportedRoutingFunctions: Pair[1],State[0..1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->State[1]}>>[*]; + supportedFilterFunctions: Pair[1],State[0..1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->State[1]}>>[*]; + supportedSimpleValueFunctions: Pair[1],State[0..1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->Pair, State>[1]}>>[*]; + supportedAggregationFunctions: Pair[1],State[0..1]->Boolean[1]}>, Function<{FunctionExpression[1], String[1]->AggregationContainer[1]}>>[*]; + supportedForPainlessScriptFunctions: Pair[1],State[0..1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->Pair[1]}>>[*]; } Class meta::external::store::elasticsearch::v7::pureToEs::TDSESDetail @@ -79,6 +80,8 @@ Class meta::external::store::elasticsearch::v7::pureToEs::TDSESDetail path(){ $this.resultPath.path() }:String[1]; + expression: FunctionExpression[0..1]; + format: String[0..1]; } function meta::external::store::elasticsearch::v7::pureToEs::processSelectAllTds(vs : FunctionExpression[1], req: State[1]): State[1] @@ -97,10 +100,12 @@ function meta::external::store::elasticsearch::v7::pureToEs::processSelectAllTds }) ->concatenate(^_IDResultPath(property = ^KeywordProperty(), fieldPath = '_id')); + let tdsDetails = $resultPaths->map(x | let type = $x.property->propertySupport().pureType->cast(@DataType); ^TDSESDetail(type = $type, name = $x.path(), resultPath = $x, format = if($type == Date, |'date_optional_time', |[]));); + ^$req( lazyIndex = $index, - search = resultPathToQuery($resultPaths, $req.search), - tdsESDetails = $resultPaths->map(x | ^TDSESDetail(type = $x.property->propertySupport().pureType->cast(@DataType), name = $x.path(), resultPath = $x)) + search = resultPathToQuery($tdsDetails, $req.search), + tdsESDetails = $tdsDetails ); } @@ -112,46 +117,75 @@ function meta::external::store::elasticsearch::v7::pureToEs::getResultPath(prope ); } -function meta::external::store::elasticsearch::v7::pureToEs::resultPathToQuery(resultPaths: ResultPath[*], search: SearchRequestBody[1]): SearchRequestBody[1] +function meta::external::store::elasticsearch::v7::pureToEs::resultPathToQuery(tdsDetails: TDSESDetail[*], search: SearchRequestBody[1]): SearchRequestBody[1] { - let frps = $resultPaths->filter(x | $x->instanceOf(FieldResultPath))->cast(@FieldResultPath); - let srps = $resultPaths->concatenate($frps.readFrom)->filter(x | $x->instanceOf(SourceFieldResultPath))->cast(@SourceFieldResultPath); - let docps = $resultPaths->concatenate($frps.readFrom)->filter(x | $x->instanceOf(DocValueResultPath))->cast(@DocValueResultPath); + let frps = $tdsDetails.resultPath->filter(x | $x->instanceOf(FieldResultPath))->cast(@FieldResultPath); + let srps = $tdsDetails.resultPath->concatenate($frps.readFrom)->filter(x | $x->instanceOf(SourceFieldResultPath))->cast(@SourceFieldResultPath); + let docps = $tdsDetails.resultPath->concatenate($frps.readFrom)->filter(x | $x->instanceOf(DocValueResultPath))->cast(@DocValueResultPath); let seachWithSrps = $srps->isNotEmpty()->if(| ^$search(_source = ^SourceConfig(filter = ^SourceFilter(includes = $srps.path()->literal()))), | $search); - let fields = $docps->map(x | $x->resultPathToFieldAndFormat()); + let fields = $docps->map(x | $x->resultPathToFieldAndFormat($tdsDetails->filter(t | $t.resultPath == $x)->first())); $fields->isEmpty()->if(|$seachWithSrps, |^$seachWithSrps(docvalue_fields = $fields)); } -function meta::external::store::elasticsearch::v7::pureToEs::resultPathToFieldAndFormat(resultPath: DocValueResultPath[1]): FieldAndFormat[1] +function meta::external::store::elasticsearch::v7::pureToEs::resultPathToFieldAndFormat(resultPath: DocValueResultPath[1], tdsEsDetail: TDSESDetail[0..1]): FieldAndFormat[1] { - let format = if($resultPath.property->propertySupport().pureType == Date, | 'date_optional_time', | []); + let format = if($resultPath.property->propertySupport().pureType == Date, | $tdsEsDetail.format->defaultIfEmpty('date_optional_time')->toOne(), | []); ^FieldAndFormat(field = $resultPath.path()->literal(), format = $format->literal()); } +function meta::external::store::elasticsearch::v7::pureToEs::processExtend(vs : FunctionExpression[1], initReq: State[1]): State[1] +{ + $initReq.debug(|'Processing ->extend'); + processProject($vs, $initReq, true); +} + function meta::external::store::elasticsearch::v7::pureToEs::processProject(vs : FunctionExpression[1], initReq: State[1]): State[1] { $initReq.debug(|'Processing ->project'); - let currReq = process($vs.parametersValues->at(0), ^$initReq(inProject = true)); + processProject($vs, $initReq, false); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processProject(vs : FunctionExpression[1], initReq: State[1], extend: Boolean[1]): State[1] +{ + let currReq = process($vs.parametersValues->at(0), $initReq); + assert(!$currReq.inFilter); assertFalse($currReq.aggregationQuery, |'project not supported in aggreagtion queries'); let cols = $vs->instanceValuesAtParameter(1, $currReq.sq.inScopeVars); - let fields = $cols->match([ - tdsCols: BasicColumnSpecification[*] | $tdsCols->map(x | $x->processProjectColumn($currReq)) + let fieldsStatePair = $cols->match([ + tdsCols: BasicColumnSpecification[*] | $tdsCols->fold( + {col: BasicColumnSpecification[1], pair: Pair, State>[1] | + let details = $col->processProjectColumn($pair.second); + pair(list($pair.first.values->concatenate($details.first)), $details.second); + }, pair(list([]->cast(@TDSESDetail)), ^$currReq(inProject = true))) ]); - let search = $currReq.search; - ^$currReq( - search = resultPathToQuery($fields.resultPath, $search), + let newReq = $fieldsStatePair.second; + let fields = if($extend, |$newReq.tdsESDetails, |[])->concatenate($fieldsStatePair.first.values); + let search = $newReq.search; + ^$newReq( + search = resultPathToQuery($fields, $search), tdsESDetails = $fields, inProject = false ); } -function meta::external::store::elasticsearch::v7::pureToEs::processProjectColumn(vs : BasicColumnSpecification[1], initReq: State[1]): TDSESDetail[1] +function meta::external::store::elasticsearch::v7::pureToEs::processProjectColumn(vs : BasicColumnSpecification[1], initReq: State[1]): Pair[1] { let expr = $vs.func->cast(@FunctionDefinition).expressionSequence->toOne('tds column projection only supports simple expressions'); let name = $vs.name; - let tdsDetail = $expr->extractSimpleValue($initReq).first.values->cast(@TDSESDetail)->toOne(); - ^$tdsDetail(name = $name, type = $vs.func->functionReturnType().rawType->toOne()->cast(@DataType)); + let type = $vs.func->functionReturnType().rawType->toOne()->cast(@DataType)->map(x | $x->_subTypeOf(Enum)->if(|String, |$x)); + + $initReq.debug(|'Processing project - %s = %s'->format([$vs.name, $expr->printValueSpecification('')])); + + $expr->extractSimpleValue($initReq).first.values->match([ + tdsDetail: TDSESDetail[1] | pair(^$tdsDetail(name = $name, type = $type), $initReq), + {fe: FunctionExpression[1] | + let resultPath = ^DocValueResultPath(fieldPath = $name, property = $type->defaultRuntimePropertyForPureType()); + let scripted = $fe->toRuntimeMapping($resultPath, $initReq); + pair(^TDSESDetail(type = $type, name = $name, resultPath = $resultPath, expression = $fe, format = $scripted.search.runtime_mappings->toOne()->get($name).format.value), $scripted); + }, + any: Any[*] | fail(|'Cannot project column - %s:%s'->format([$name,$any->type()->elementToPath()]))->cast(@Pair) + ]); } function meta::external::store::elasticsearch::v7::pureToEs::processLimit(vs : FunctionExpression[1], initReq: State[1]): State[1] @@ -165,7 +199,7 @@ function meta::external::store::elasticsearch::v7::pureToEs::processLimit(vs : F } -function meta::external::store::elasticsearch::v7::pureToEs::processDrop(vs : FunctionExpression[1], initReq: State[1]): State[1] +function meta::external::store::elasticsearch::v7::pureToEs::processDrop(vs : FunctionExpression[1], initReq: State[1]): State[1] { $initReq.debug(|'Processing ->drop'); let currReq = process($vs.parametersValues->at(0), $initReq); @@ -239,12 +273,12 @@ function meta::external::store::elasticsearch::v7::pureToEs::processGroupBy(vs: assert($groupByCols->size() == $groupByTdsESDetails->size(), | 'grouping by unknown columns: ' + $groupByCols->removeAll($groupByTdsESDetails.name)->joinStrings('[', ', ', ']')); let aggPairs = $aggregateValues->map({x | - let rawToAggregate = processProjectColumn(^BasicColumnSpecification(func = $x.mapFn, name = $x.name), $groupedReq); + let rawToAggregate = processProjectColumn(^BasicColumnSpecification(func = $x.mapFn, name = $x.name), $groupedReq).first; let aggFunc = $x.aggregateFn->deepByPassRouterInfo()->cast(@FunctionDefinition).expressionSequence->toOne('tds aggregation only supports simple expressions: max, min, sum, etc.'); let container = if ($aggFunc->instanceOf(FunctionExpression), | - $groupedReq.supportedAggregationFunctions->findAndEvalSupportedFunction($aggFunc->cast(@FunctionExpression), $rawToAggregate.path()) + $groupedReq.supportedAggregationFunctions->findAndEvalSupportedFunction($aggFunc->cast(@FunctionExpression), $rawToAggregate.path(), $groupedReq) , | fail('Unsupported aggregation functions: ' + $aggFunc->printValueSpecification('')); @@ -294,27 +328,25 @@ function meta::external::store::elasticsearch::v7::pureToEs::processGroupBy(vs: function meta::external::store::elasticsearch::v7::pureToEs::processFilter(vs : FunctionExpression[1], initReq: State[1]): State[1] { $initReq.debug(|'Processing ->filter'); - let req = ^$initReq(inFilter = true); - let debug = $req.debug; - let currReq = process($vs.parametersValues->at(0), ^$req(debug = ^$debug(space = $debug.space + '\t'))); + let debug = $initReq.debug; + let currReq = process($vs.parametersValues->at(0), ^$initReq(debug = ^$debug(space = $debug.space + '\t'))); assertFalse($currReq.aggregationQuery, |'filter not supported in aggregation queries'); - let withFilterReq = processFilterLambda($vs.parametersValues->at(1), $currReq); + let withFilterReq = processFilterLambda($vs.parametersValues->at(1), ^$currReq(inFilter = true)); ^$withFilterReq(inFilter = false, debug = $debug); } function meta::external::store::elasticsearch::v7::pureToEs::processFilterLambda(vs : ValueSpecification[1], req: State[1]): State[1] { $req.debug(|'-- processing filter lambda ' + $vs->type()->toString() + ': ' + $vs->printValueSpecification('\n')); - $vs->match([ fr: FunctionRoutedValueSpecification[1] | $fr.value->processFilterLambda($req), iv: InstanceValue[1] | $iv.values->match([ - f: FunctionDefinition[1 ] | $f.expressionSequence->at(0)->processFilterLambda($req), - any: Any[1] | fail('not supported: ' + $iv.values->type()->toString());$req; + f: FunctionDefinition[1] | $f.expressionSequence->at(0)->processFilterLambda($req), + any: Any[1] | fail('not supported: ' + $iv.values->type()->toString())->cast(@State) ]), - fe: FunctionExpression[1] | $req.supportedFilterFunctions->findAndEvalSupportedFunction($fe, $req), - any: Any[1] | fail('not supported: ' + $any->printValueSpecification(''));$req; + fe: FunctionExpression[1] | $req.supportedFilterFunctions->findAndEvalSupportedFunction($fe, $req, $req), + any: Any[1] | fail('not supported: ' + $any->printValueSpecification(''))->cast(@State) ]); } @@ -472,18 +504,18 @@ function <> meta::external::store::elasticsearch::v7::pureToEs:: function meta::external::store::elasticsearch::v7::pureToEs::process(vs : ValueSpecification[1], req: State[1]): State[1] { - $req.debug(|'-- processing VS ' + $vs->type()->toString() + ': ' + $vs->printValueSpecification('\n')); + $req.debug(|'-- processing VS ' + $vs->printValueSpecification('\n')); $vs->match([ - fe: FunctionExpression[1] | $req.supportedRoutingFunctions->findAndEvalSupportedFunction($fe, $req), + fe: FunctionExpression[1] | $req.supportedRoutingFunctions->findAndEvalSupportedFunction($fe, $req, $req), tds: TDSRoutedValueSpecification[1] | $tds.value->process($req), ervs: ExtendedRoutedValueSpecification[1] | $ervs.value->process($req), any: Any[1] | fail('not supported: ' + $any->type()->toString());$req; ]); } -function meta::external::store::elasticsearch::v7::pureToEs::findAndEvalSupportedFunction(funcs: Pair[1]->Boolean[1]}>, Function<{FunctionExpression[1], P[n]->T[m]}>>[*], fe : FunctionExpression[1], req: P[n]): T[m] +function meta::external::store::elasticsearch::v7::pureToEs::findAndEvalSupportedFunction(funcs: Pair[1],State[0..1]->Boolean[1]}>, Function<{FunctionExpression[1], P[n]->T[m]}>>[*], fe : FunctionExpression[1], req: P[n], state: State[1]): T[m] { - eval($funcs->filter(x|$x.first->eval($fe.func))->at(0)->toOne('function not supported yet: ' + $fe.func->elementToPath()).second, $fe, $req); + eval($funcs->filter(x|$x.first->eval($fe.func, $state))->first()->toOne('function not supported yet: ' + $fe.func->elementToPath()).second, $fe, $req); } function meta::external::store::elasticsearch::v7::pureToEs::defaultSize(): Integer[1] @@ -507,7 +539,8 @@ function meta::external::store::elasticsearch::v7::pureToEs::process(sq:StoreQue supportedFilterFunctions = supportedFilterFunctions(), supportedRoutingFunctions = supportedRoutingFunctions(), supportedSimpleValueFunctions = supportedSimpleValueFunctions(), - supportedAggregationFunctions = supportedAggregationFunctions() + supportedAggregationFunctions = supportedAggregationFunctions(), + supportedForPainlessScriptFunctions = supportedForPainlessScriptFunctions() ); let processedReq = $sq.fe->process($req); @@ -515,30 +548,38 @@ function meta::external::store::elasticsearch::v7::pureToEs::process(sq:StoreQue ^$processedReq(search = if($search.size->isEmpty(), | ^$search(size = defaultSize()->literal()), | $search)); } -function meta::external::store::elasticsearch::v7::pureToEs::supportedRoutingFunctions():Pair[1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->State[1]}>>[*] +function meta::external::store::elasticsearch::v7::pureToEs::supportedIfEqual(func: Function[1]):LambdaFunction<{Function[1],State[0..1]->Boolean[1]}>[1] +{ + {x: Function[1], req: State[0..1] | $x == $func} +} + +function meta::external::store::elasticsearch::v7::pureToEs::supportedRoutingFunctions():Pair[1],State[0..1]->Boolean[1]}>, Function<{FunctionExpression[1],State[1]->State[1]}>>[*] { let supported = [ - pair(x: Function[1] | $x == indexToTDS_Elasticsearch7Store_1__String_1__TabularDataSet_1_, processSelectAllTds_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::project_TabularDataSet_1__ColumnSpecification_MANY__TabularDataSet_1_, processProject_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(indexToTDS_Elasticsearch7Store_1__String_1__TabularDataSet_1_), processSelectAllTds_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::project_TabularDataSet_1__ColumnSpecification_MANY__TabularDataSet_1_), processProject_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::extend_TabularDataSet_1__BasicColumnSpecification_MANY__TabularDataSet_1_), processExtend_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::filter_TabularDataSet_1__Function_1__TabularDataSet_1_, processFilter_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::filter_TabularDataSet_1__Function_1__TabularDataSet_1_), processFilter_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::sort_TabularDataSet_1__String_MANY__TabularDataSet_1_, processDefaultSort_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::sort_TabularDataSet_1__SortInformation_MANY__TabularDataSet_1_, processSortWithInformation_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::sort_TabularDataSet_1__String_1__SortDirection_1__TabularDataSet_1_, processSortWithDirection_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::sort_TabularDataSet_1__String_MANY__TabularDataSet_1_), processDefaultSort_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::sort_TabularDataSet_1__SortInformation_MANY__TabularDataSet_1_), processSortWithInformation_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::sort_TabularDataSet_1__String_1__SortDirection_1__TabularDataSet_1_), processSortWithDirection_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::groupBy_TabularDataSet_1__String_MANY__AggregateValue_MANY__TabularDataSet_1_, processGroupBy_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::groupBy_TabularDataSet_1__String_MANY__AggregateValue_MANY__TabularDataSet_1_), processGroupBy_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::limit_TabularDataSet_1__Integer_1__TabularDataSet_1_, processLimit_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::take_TabularDataSet_1__Integer_1__TabularDataSet_1_, processLimit_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::drop_TabularDataSet_1__Integer_1__TabularDataSet_1_, processDrop_FunctionExpression_1__State_1__State_1_), - pair(x: Function[1] | $x == meta::pure::tds::slice_TabularDataSet_1__Integer_1__Integer_1__TabularDataSet_1_, processSlice_FunctionExpression_1__State_1__State_1_) + pair(supportedIfEqual(meta::pure::tds::limit_TabularDataSet_1__Integer_1__TabularDataSet_1_), processLimit_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::take_TabularDataSet_1__Integer_1__TabularDataSet_1_), processLimit_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::drop_TabularDataSet_1__Integer_1__TabularDataSet_1_), processDrop_FunctionExpression_1__State_1__State_1_), + pair(supportedIfEqual(meta::pure::tds::slice_TabularDataSet_1__Integer_1__Integer_1__TabularDataSet_1_), processSlice_FunctionExpression_1__State_1__State_1_) ]; } function meta::external::store::elasticsearch::v7::pureToEs::processEqual(vs : FunctionExpression[1], initReq: State[1]): State[1] { + $initReq.debug(|'-- processing equal ' + $vs->printValueSpecification('\n')); + assert($initReq.inFilter, 'equal only supporter on filter context'); let leftSidePair = $vs.parametersValues->at(0)->extractSimpleValue($initReq); @@ -549,6 +590,12 @@ function meta::external::store::elasticsearch::v7::pureToEs::processEqual(vs : F let req = $rightSidePair.second; + processEqual($leftSide, $rightSide, $req); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processEqual(leftSide: Any[*], rightSide: Any[*], req: State[1]): State[1] +{ + let eqInputs = if($leftSide->size() == 1 && $leftSide->toOne()->instanceOf(TDSESDetail), |pair($leftSide->toOne()->cast(@TDSESDetail), list($rightSide)), | @@ -791,6 +838,18 @@ function meta::external::store::elasticsearch::v7::pureToEs::processIsNotEmpty(v ^$req(search = ^$search(query = ^QueryContainer(bool = ^BoolQuery(filter = $query)))); } +function meta::external::store::elasticsearch::v7::pureToEs::processTdsGetBoolean(vs : FunctionExpression[1], initReq: State[1]): State[1] +{ + assert($initReq.inFilter, 'TdsRow.getBoolean only supporter on filter context'); + + let columnPair = $vs->extractSimpleValue($initReq); + let column = $columnPair.first.values->cast(@TDSESDetail)->toOne(); + + let req = $columnPair.second; + + processEqual($column, true, $req); +} + function meta::external::store::elasticsearch::v7::pureToEs::processTdsIsNull(vs : FunctionExpression[1], initReq: State[1]): State[1] { assert($initReq.inFilter, 'TdsRow.isNull only supporter on filter context'); @@ -931,40 +990,40 @@ function meta::external::store::elasticsearch::v7::pureToEs::processNot(vs : Fun function meta::external::store::elasticsearch::v7::pureToEs::processOr(vs : FunctionExpression[1], req: State[1]): State[1] { assert($req.inFilter, 'or only supporter on filter context'); + let search = $req.search; let leftSide = $vs.parametersValues->at(0)->processFilterLambda($req); - let rightSide = $vs.parametersValues->at(1)->processFilterLambda($req); + let rightSide = $vs.parametersValues->at(1)->processFilterLambda(^$leftSide(search = $search)); let bool = ^BoolQuery(minimum_should_match = literal('1'), should = [ $leftSide.search.query->toOne(), $rightSide.search.query->toOne() ]); let query = ^QueryContainer(bool = ^BoolQuery(filter = ^QueryContainer(bool = $bool))); - let search = $req.search; - ^$req(search = ^$search(query = $query), extraNodes = $req.extraNodes->concatenate($leftSide.extraNodes)->concatenate($rightSide.extraNodes)); + ^$rightSide(search = ^$search(query = $query), extraNodes = $req.extraNodes->concatenate($leftSide.extraNodes)->concatenate($rightSide.extraNodes)); } function meta::external::store::elasticsearch::v7::pureToEs::processAnd(vs : FunctionExpression[1], req: State[1]): State[1] { assert($req.inFilter, 'and only supporter on filter context'); + let search = $req.search; let leftSide = $vs.parametersValues->at(0)->processFilterLambda($req); - let rightSide = $vs.parametersValues->at(1)->processFilterLambda($req); + let rightSide = $vs.parametersValues->at(1)->processFilterLambda(^$leftSide(search = $search)); let bool = ^BoolQuery(must = [ $leftSide.search.query->toOne(), $rightSide.search.query->toOne() ]); let query = ^QueryContainer(bool = ^BoolQuery(filter = ^QueryContainer(bool = $bool))); - let search = $req.search; - ^$req(search = ^$search(query = $query), extraNodes = $req.extraNodes->concatenate($leftSide.extraNodes)->concatenate($rightSide.extraNodes)); + ^$rightSide(search = ^$search(query = $query), extraNodes = $req.extraNodes->concatenate($leftSide.extraNodes)->concatenate($rightSide.extraNodes)); } function meta::external::store::elasticsearch::v7::pureToEs::extractSimpleValue(vs : Any[1], req: State[1]): Pair, State>[1] { - $req.debug(|'-- processing simple value ' + $vs->type()->toString()); + $req.debug(|'-- processing simple value ' + $vs->match([vs: ValueSpecification[1] | $vs->printValueSpecification(''), a:Any[1] | $a->toString()])); $vs->match([ rfe: FunctionExpression[1] | let fe = $rfe->deepByPassRouterInfo()->cast(@FunctionExpression); if($fe.func->instanceOf(QualifiedProperty), - | pair($fe.func->cast(@QualifiedProperty)->qualifiedPropertyToTDSESDetail($fe.parametersValues, $req)->list(), $req), + | $fe->qualifiedPropertyToTDSESDetail($req), | if(toOne_T_MANY__T_1_ == $fe.func, | $fe.parametersValues->at(0)->extractSimpleValue($req), - | $req.supportedSimpleValueFunctions->findAndEvalSupportedFunction($fe, $req) + | $req.supportedSimpleValueFunctions->findAndEvalSupportedFunction($fe, $req, $req) ));, iv: InstanceValue[1] | $iv.values->fold({x, currState | let r = $x->extractSimpleValue($currState.second); @@ -976,124 +1035,134 @@ function meta::external::store::elasticsearch::v7::pureToEs::extractSimpleValue( ]); } -function meta::external::store::elasticsearch::v7::pureToEs::qualifiedPropertyToTDSESDetail(qp : QualifiedProperty[1], params: ValueSpecification[*], req: State[1]): TDSESDetail[1] +function meta::external::store::elasticsearch::v7::pureToEs::qualifiedPropertyToTDSESDetail(fe : FunctionExpression[1], req: State[1]): Pair, State>[1] { + let qp = $fe.func->cast(@QualifiedProperty); + let params = $fe.parametersValues; assert($params->at(0)->byPassRouterInfo().genericType.rawType->toOne() == TDSRow, 'Only TDSRow operations supported'); - let tdsProperties = ['getNumber', 'getInteger', 'getString', 'getNullableString', 'getFloat', 'getDate', 'getBoolean', 'getEnum', 'getDateTime', 'getStrictDate', 'isNull', 'isNotNull']; + let tdsProperties = ['getNumber', 'getInteger', 'getString', 'getNullableString', 'getFloat', 'getDate', 'getBoolean', 'getEnum', 'getDateTime', 'getStrictDate'] + ->concatenate(if($req.inProject, |[], |['isNull', 'isNotNull'])); let funcName = $qp.functionName->toOne(); - let valid = $tdsProperties->contains($funcName); - assert($tdsProperties->contains($funcName), | 'Unsupported TDSRow function:' + $qp.functionName->toOne()); + $tdsProperties->contains($funcName)->if( + {| + let propertyName = $params->at(1)->match([ + iv:InstanceValue[1]|$iv.values, + any: Any[1] | fail($any->printValueSpecification('\n') + ' not supported'); + ])->toOne()->toString(); - let propertyName = $params->at(1)->match([ - iv:InstanceValue[1]|$iv.values, - any: Any[1] | fail($any->printValueSpecification('\n') + ' not supported'); - ])->toOne()->toString(); + let tdsDetail = $req.tdsESDetails->filter(f | $f.name == $propertyName) + ->toOne('Property \'%s\' not found on query project. Available: [%s]'->format([$propertyName, $req.tdsESDetails.name->joinStrings('\'', '\', \'', '\'')])); - $req.tdsESDetails->filter(f | $f.name == $propertyName) - ->toOne('Property \'%s\' not found on query project. Available: [%s]'->format([$propertyName, $req.tdsESDetails.name->joinStrings('\'', '\', \'', '\'')])); + pair($tdsDetail->list(), $req); + }, + {| + $req.supportedSimpleValueFunctions->findAndEvalSupportedFunction($fe, $req, $req); + } + ); } -function meta::external::store::elasticsearch::v7::pureToEs::supportedFilterFunctions():Pair[1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->State[1]}>>[*] +function meta::external::store::elasticsearch::v7::pureToEs::supportedFilterFunctions():Pair[1],State[1]->Boolean[1]}>, Function<{FunctionExpression[1],State[1]->State[1]}>>[*] { let supported = [ - pair(x: Function[1] | $x == meta::pure::functions::boolean::equal_Any_MANY__Any_MANY__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processEqual_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::or_Boolean_1__Boolean_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processOr_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::and_Boolean_1__Boolean_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processAnd_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::not_Boolean_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processNot_FunctionExpression_1__State_1__State_1_) + pair(supportedIfEqual(meta::pure::functions::boolean::equal_Any_MANY__Any_MANY__Boolean_1_), processEqual_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::or_Boolean_1__Boolean_1__Boolean_1_), processOr_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::and_Boolean_1__Boolean_1__Boolean_1_), processAnd_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::not_Boolean_1__Boolean_1_), processNot_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::greaterThan_Number_1__Number_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processGreaterThan_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::greaterThanEqual_Number_1__Number_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processGreaterThanEqual_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::greaterThan_Number_1__Number_1__Boolean_1_), processGreaterThan_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::greaterThanEqual_Number_1__Number_1__Boolean_1_), processGreaterThanEqual_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::greaterThan_Date_1__Date_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processGreaterThan_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::greaterThanEqual_Date_1__Date_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processGreaterThanEqual_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::greaterThan_Date_1__Date_1__Boolean_1_), processGreaterThan_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::greaterThanEqual_Date_1__Date_1__Boolean_1_), processGreaterThanEqual_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::lessThan_Number_1__Number_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processLessThan_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::lessThanEqual_Number_1__Number_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processLessThanEqual_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::lessThan_Number_1__Number_1__Boolean_1_), processLessThan_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::lessThanEqual_Number_1__Number_1__Boolean_1_), processLessThanEqual_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::lessThan_Date_1__Date_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processLessThan_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::boolean::lessThanEqual_Date_1__Date_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processLessThanEqual_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::lessThan_Date_1__Date_1__Boolean_1_), processLessThan_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::boolean::lessThanEqual_Date_1__Date_1__Boolean_1_), processLessThanEqual_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x->instanceOf(QualifiedProperty) && $x->cast(@QualifiedProperty).name == 'isNull', meta::external::store::elasticsearch::v7::pureToEs::processTdsIsNull_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::collection::isEmpty_Any_$0_1$__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processIsEmpty_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x->instanceOf(QualifiedProperty) && $x->cast(@QualifiedProperty).name == 'isNotNull', meta::external::store::elasticsearch::v7::pureToEs::processTdsIsNotNull_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::collection::isNotEmpty_Any_$0_1$__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processIsNotEmpty_FunctionExpression_1__State_1__State_1_) + ,pair({x: Function[1], req: State[1] | $x->instanceOf(QualifiedProperty) && $x->cast(@QualifiedProperty).name == 'getBoolean'}, processTdsGetBoolean_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::string::startsWith_String_1__String_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processStartsWith_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::string::endsWith_String_1__String_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processEndsWith_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::string::contains_String_$0_1$__String_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processContains_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::string::contains_String_1__String_1__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processContains_FunctionExpression_1__State_1__State_1_) - ,pair(x: Function[1] | $x == meta::pure::functions::collection::in_Any_1__Any_MANY__Boolean_1_, meta::external::store::elasticsearch::v7::pureToEs::processIn_FunctionExpression_1__State_1__State_1_) + ,pair({x: Function[1], req: State[1] | $x->instanceOf(QualifiedProperty) && $x->cast(@QualifiedProperty).name == 'isNull'}, processTdsIsNull_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::collection::isEmpty_Any_$0_1$__Boolean_1_), processIsEmpty_FunctionExpression_1__State_1__State_1_) + ,pair({x: Function[1], req: State[1] | $x->instanceOf(QualifiedProperty) && $x->cast(@QualifiedProperty).name == 'isNotNull'}, processTdsIsNotNull_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::collection::isNotEmpty_Any_$0_1$__Boolean_1_), processIsNotEmpty_FunctionExpression_1__State_1__State_1_) + + ,pair(supportedIfEqual(meta::pure::functions::string::startsWith_String_1__String_1__Boolean_1_), processStartsWith_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::string::endsWith_String_1__String_1__Boolean_1_), processEndsWith_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::string::contains_String_$0_1$__String_1__Boolean_1_), processContains_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::string::contains_String_1__String_1__Boolean_1_), processContains_FunctionExpression_1__State_1__State_1_) + ,pair(supportedIfEqual(meta::pure::functions::collection::in_Any_1__Any_MANY__Boolean_1_), processIn_FunctionExpression_1__State_1__State_1_) ]; } -function meta::external::store::elasticsearch::v7::pureToEs::supportedSimpleValueFunctions(): Pair[1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->Pair, State>[1]}>>[*] +function meta::external::store::elasticsearch::v7::pureToEs::supportedSimpleValueFunctions(): Pair[1], State[1]->Boolean[1]}>, Function<{FunctionExpression[1], State[1]->Pair, State>[1]}>>[*] { let supported = [ - pair(x: Function[1] | $x == meta::pure::functions::lang::extractEnumValue_Enumeration_1__String_1__T_1_, meta::external::store::elasticsearch::v7::pureToEs::reactivateFE_FunctionExpression_1__State_1__Pair_1_) - ,pair(x: Function[1] | $x == meta::pure::tds::asc_String_1__SortInformation_1_, meta::external::store::elasticsearch::v7::pureToEs::reactivateFE_FunctionExpression_1__State_1__Pair_1_) - ,pair(x: Function[1] | $x == meta::pure::tds::desc_String_1__SortInformation_1_, meta::external::store::elasticsearch::v7::pureToEs::reactivateFE_FunctionExpression_1__State_1__Pair_1_) - ,pair(x: Function[1] | $x == meta::pure::tds::agg_String_1__FunctionDefinition_1__FunctionDefinition_1__AggregateValue_1_, meta::external::store::elasticsearch::v7::pureToEs::reactivateFE_FunctionExpression_1__State_1__Pair_1_) - // catch all - delegate to platform evaluation of expression, and assign to variable. ES will then use the variable on the plan - ,pair(x: Function[1] | true, meta::external::store::elasticsearch::v7::pureToEs::evaluateAsPureExpressionNode_FunctionExpression_1__State_1__Pair_1_) + pair(supportedIfEqual(meta::pure::functions::lang::extractEnumValue_Enumeration_1__String_1__T_1_), reactivateFE_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(meta::pure::tds::asc_String_1__SortInformation_1_), reactivateFE_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(meta::pure::tds::desc_String_1__SortInformation_1_), reactivateFE_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(meta::pure::tds::agg_String_1__FunctionDefinition_1__FunctionDefinition_1__AggregateValue_1_), reactivateFE_FunctionExpression_1__State_1__Pair_1_) + // catch all + ,pair({x: Function[1], req: State[1] | $req.inFilter}, evaluateAsPureExpressionNode_FunctionExpression_1__State_1__Pair_1_) + ,pair({x: Function[1], req: State[1] | $req.inProject}, passthruForPainlessExpression_FunctionExpression_1__State_1__Pair_1_) ]; } function meta::external::store::elasticsearch::v7::pureToEs::reactivateFE(fe: FunctionExpression[1], req: State[1]): Pair, State>[1] { + $req.debug(|'Reactivating FE: ' + $fe->printValueSpecification('')); pair($fe->cast(@FunctionExpression)->reactivate($req.sq.inScopeVars)->list(), $req); } +function meta::external::store::elasticsearch::v7::pureToEs::passthruForPainlessExpression(fe: FunctionExpression[1], req: State[1]): Pair, State>[1] +{ + $req.debug(|'Passthru for painless: ' + $fe->printValueSpecification('')); + assert($req.inProject, |'Evaluation as runtime mapping expression only supported on project context'); + pair(list($fe), $req); +} + function meta::external::store::elasticsearch::v7::pureToEs::evaluateAsPureExpressionNode(fe: FunctionExpression[1], req: State[1]): Pair, State>[1] { + $req.debug(|'Converting to pure expression node: ' + $fe->printValueSpecification('')); + assert($req.inFilter, |'Evaluation as pure expression only supported on filter context'); let type = $fe.func->functionReturnType(); let multiplicity = $fe.func->functionReturnMultiplicity(); - // todo any validation to prevent compilation errors?? - // let variables = $fe->findVariableExpressionsInValueSpecification(); - - let expression = $fe->printFunctionExpression(''); - let codeHash = $expression->hash(HashType.SHA256); - let charToIntMap = newMap([ - pair('0', 0), pair('1', 1), pair('2', 2), pair('3', 3), - pair('4', 4), pair('5', 5), pair('6', 6), pair('7', 7), - pair('8', 8), pair('9', 9), pair('a', 10), pair('b', 11), - pair('c', 12), pair('d', 13), pair('e', 14), pair('f', 15) - ]); - let prime = floor(pow(2, 31) - 1); - let varName = '@' + $codeHash->toLower()->chunk(1)->fold({c, h | (31 * $h) + $charToIntMap->get($c)->toOne() + 47}, 0)->mod($prime)->toString(); + let varName = '@planVariable' + $req.counter->toString(); let platformNode = processValueSpecification($fe, ^PlatformPlanGenerationState(inScopeVars = $req.sq.inScopeVars, exeCtx = $req.exeCtx), $req.extensions, $req.debug) - ->toOne('Expression currently not supported: ' + $expression); + ->toOne('Expression currently not supported: ' + $fe->printFunctionExpression('')); let allocNode = ^AllocationExecutionNode(resultType = $platformNode.resultType, varName = $varName, executionNodes = $platformNode); pair( list(^PlanVarPlaceHolder(type = $platformNode.resultType.type, name = $varName, multiplicity = $fe.func->functionReturnMultiplicity())), - ^$req(extraNodes = $req.extraNodes->concatenate($allocNode)) + ^$req(extraNodes = $req.extraNodes->concatenate($allocNode), counter = $req.counter + 1) ); } -function meta::external::store::elasticsearch::v7::pureToEs::supportedAggregationFunctions(): Pair[1]->Boolean[1]}>, Function<{FunctionExpression[1], String[1]->AggregationContainer[1]}>>[*] +function meta::external::store::elasticsearch::v7::pureToEs::supportedAggregationFunctions(): Pair[1],State[1]->Boolean[1]}>, Function<{FunctionExpression[1],String[1]->AggregationContainer[1]}>>[*] { let supported = [ - pair(x: Function[1] | $x == meta::pure::functions::math::sum_Integer_MANY__Integer_1_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(sum = ^SumAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::sum_Float_MANY__Float_1_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(sum = ^SumAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::sum_Number_MANY__Number_1_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(sum = ^SumAggregation(field = $y->literal()))}) + pair(supportedIfEqual(meta::pure::functions::math::sum_Integer_MANY__Integer_1_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(sum = ^SumAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::sum_Float_MANY__Float_1_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(sum = ^SumAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::sum_Number_MANY__Number_1_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(sum = ^SumAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::max_Integer_MANY__Integer_$0_1$_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(max = ^MaxAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::max_Float_MANY__Float_$0_1$_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(max = ^MaxAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::max_Number_MANY__Number_$0_1$_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(max = ^MaxAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::max_Integer_MANY__Integer_$0_1$_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(max = ^MaxAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::max_Float_MANY__Float_$0_1$_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(max = ^MaxAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::max_Number_MANY__Number_$0_1$_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(max = ^MaxAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::min_Integer_MANY__Integer_$0_1$_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(min = ^MinAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::min_Float_MANY__Float_$0_1$_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(min = ^MinAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::min_Number_MANY__Number_$0_1$_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(min = ^MinAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::min_Integer_MANY__Integer_$0_1$_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(min = ^MinAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::min_Float_MANY__Float_$0_1$_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(min = ^MinAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::min_Number_MANY__Number_$0_1$_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(min = ^MinAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::average_Integer_MANY__Float_1_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(avg = ^AverageAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::average_Float_MANY__Float_1_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(avg = ^AverageAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::math::average_Number_MANY__Float_1_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(avg = ^AverageAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::average_Integer_MANY__Float_1_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(avg = ^AverageAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::average_Float_MANY__Float_1_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(avg = ^AverageAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::math::average_Number_MANY__Float_1_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(avg = ^AverageAggregation(field = $y->literal()))}) - ,pair(x: Function[1] | $x == meta::pure::functions::collection::count_Any_MANY__Integer_1_, {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(value_count = ^ValueCountAggregation(field = $y->literal()))}) + ,pair(supportedIfEqual(meta::pure::functions::collection::count_Any_MANY__Integer_1_), {x: FunctionExpression[1], y: String[1] | ^AggregationContainer(value_count = ^ValueCountAggregation(field = $y->literal()))}) ]; } @@ -1107,7 +1176,9 @@ function meta::external::store::elasticsearch::v7::pureToEs::literalOrExpression date: Date[1] | $date->toString()->literal(), var: VariableExpression[1] | expression(^PlanVarPlaceHolder(name = $var.name, type = $var.genericType.rawType->toOne(), multiplicity = $var.multiplicity), $supportZeroMult), planVar: PlanVarPlaceHolder[1] | $planVar->expression($supportZeroMult), + enum: Enum[1] | $enum->toString()->literal(), none: Any[0] | [], + a: Any[1] | fail(|'not supported type: ' + $a->type()->elementToPath())->literal(), mult: Any[*] | $mult->map(x | $x->literalOrExpression($supportZeroMult)); ]); } @@ -1154,4 +1225,400 @@ function meta::external::store::elasticsearch::v7::pureToEs::varFreemarkerExpres | '${' + $var.name + $toJson + '}' )); +} + +function meta::external::store::elasticsearch::v7::pureToEs::toRuntimeMapping(vs: FunctionExpression[1], field: DocValueResultPath[1], initReq: State[1]): State[1] +{ + let p = processPainless($vs, ^$initReq(inProject = false, inFilter = true)); + + let script = $p.first; + let req = $p.second; + + let type = $field.property->match([ + b: BooleanProperty[1] | RuntimeFieldType._boolean, + k: KeywordProperty[1] | RuntimeFieldType.keyword, + l: LongNumberProperty[1] | RuntimeFieldType._long, + d: DoubleNumberProperty[1] | RuntimeFieldType._double, + b: DateProperty[1] | RuntimeFieldType.date + ]); + + let toEpochMillis = if ($type == RuntimeFieldType.date, |'.toInstant().toEpochMilli()', |''); + + let emittingScript = ^$script(source = 'emit(%s%s)'->format([$script.source.value->toOne(), $toEpochMillis])->literal()); + + let format = painlessDateFunctionsFormat($vs.func)->literal(); + + let newField = pair($field.path(), ^RuntimeField(script = ^Script(inline = $emittingScript), format = $format, type = $type)); + + let search = $req.search; + let fields = $search.runtime_mappings->map(x | $x->keyValues()); + + let newSearch = ^$search(runtime_mappings = newMap($fields->concatenate($newField))); + ^$req(search = $newSearch, inProject = $initReq.inProject, inFilter = $initReq.inFilter); +} + +function meta::external::store::elasticsearch::v7::pureToEs::painlessParameter(req: State[1]): Pair[1] +{ + pair('param' + $req.counter->toString(), ^$req(counter = $req.counter + 1)); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainless(vs: ValueSpecification[1], initReq: State[1]): Pair[1] +{ + $vs->match([ + fe: FunctionExpression[1] | $initReq.supportedForPainlessScriptFunctions->findAndEvalSupportedFunction($fe, $initReq, $initReq), + iv: InstanceValue[1] | + $iv.values->match([ + f: FunctionDefinition[1] | $f.expressionSequence->at(0)->processPainless($initReq), + any: Any[1] | $any->processPainlessScalarValue($initReq) + ]) + ]); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessScalarValue(any: Any[1], req: State[1]): Pair[1] +{ + let value = $any->literalOrExpression(false)->toOne(); + + let param = painlessParameter($req); + let script = 'params[\'%s\']'->format($param.first); + + pair(^InlineScript(source = $script->literal(), params = newMap(pair($param.first, $value))), $param.second); +} + +function meta::external::store::elasticsearch::v7::pureToEs::painlessExtractField(tdsESDetail: TDSESDetail[1]): String[1] +{ + $tdsESDetail.resultPath.property->match([ + text: TextProperty[1] | 'params[\'_source\'][\'%s\']', + any: Any[1] | 'doc[\'%s\'].value'; + ])->format($tdsESDetail.path()); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessEnumValue(fe: FunctionExpression[1], req: State[1]): Pair[1] +{ + $fe.parametersValues->at(1)->processPainless($req); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessIsEmpty(fe: FunctionExpression[1], initReq: State[1]): Pair[1] +{ + let fieldPair = if ($fe.func->instanceOf(QualifiedProperty), + |$fe->qualifiedPropertyToTDSESDetail($initReq), + |$fe.parametersValues->at(0)->extractSimpleValue($initReq) + ); + let field = $fieldPair.first.values->cast(@TDSESDetail)->toOne('isEmpty only works on tds columns for now'); + let req = $fieldPair.second; + + let script = $field.resultPath.property->match([ + text: TextProperty[1] | 'params[\'_source\'][\'%s\'] == null', + any: Any[1] | 'doc[\'%s\'].size() == 0'; + ])->format($field.path()); + + pair(^InlineScript(source = $script->literal(), params = newMap([])), $req); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessIsNotEmpty(fe: FunctionExpression[1], initReq: State[1]): Pair[1] +{ + let fieldPair = if ($fe.func->instanceOf(QualifiedProperty), + |$fe->qualifiedPropertyToTDSESDetail($initReq), + |$fe.parametersValues->at(0)->extractSimpleValue($initReq) + ); + + let field = $fieldPair.first.values->cast(@TDSESDetail)->toOne('isNotEmpty only works on tds columns for now'); + let req = $fieldPair.second; + + let script = $field.resultPath.property->match([ + text: TextProperty[1] | 'params[\'_source\'][\'%s\'] != null', + any: Any[1] | 'doc[\'%s\'].size() != 0'; + ])->format($field.path()); + + pair(^InlineScript(source = $script->literal(), params = newMap([])), $req); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessBooleanComparison(fe: FunctionExpression[1], initReq: State[1]): Pair[1] +{ + let funcName = $fe.func.functionName->toOne(); + let maybeOperation = [ + pair('equal', '=='), + pair('lessThanEqual', '<='), + pair('lessThan', '<'), + pair('greaterThanEqual', '>='), + pair('greaterThan', '>') + ]->filter(x | $x.first == $funcName).second->toOne('Boolean comparison not supported: ' + $funcName); + + let leftSidePair = $fe.parametersValues->at(0)->extractSimpleValue($initReq); + let leftSide = $leftSidePair.first.values->toOne(); + + let rightSidePair = $fe.parametersValues->at(1)->extractSimpleValue($leftSidePair.second); + let rightSide = $rightSidePair.first.values->toOne(); + + let inputs = if($leftSide->size() == 1 && $leftSide->toOne()->instanceOf(TDSESDetail), + |pair($leftSide->toOne()->cast(@TDSESDetail), list($rightSide)), + | + assert($rightSide->size() == 1 && $rightSide->toOne()->instanceOf(TDSESDetail), 'should include an index property'); + pair($rightSide->toOne()->cast(@TDSESDetail), list($leftSide)); + ); + + let operation = if($rightSide->size() == 1 && $rightSide->toOne()->instanceOf(TDSESDetail) && $maybeOperation != '==', + | + [ + pair('<=', '>='), + pair('<', '>'), + pair('>=', '<='), + pair('>', '<') + ]->filter(x | $x.first == $maybeOperation).second->toOne(), + |$maybeOperation + ); + + let tdsESDetail = $inputs.first; + let value = $inputs.second.values->toOne(); + + let lastReq = $rightSidePair.second; + let param = painlessParameter($lastReq); + + $tdsESDetail.type->in([Date, StrictDate, DateTime])->if( + {| + let newValue = $value->match([ + date: Date[1] | $date->toEpochValue(DurationUnit.MILLISECONDS), + var: VariableExpression[1] | ^PlanVarPlaceHolder(name = $var.name, type = $var.genericType.rawType->toOne(), multiplicity = $var.multiplicity), + pv: PlanVarPlaceHolder[1] | $pv, // $pv.type + any: Any[*] | fail() + ]); + + let paramScript = $newValue->match([ + pv: PlanVarPlaceHolder[1] | + if($pv.type == StrictDate, + |'LocalDate.parse(params[\'%s\']).atTime(0, 0).toInstant(ZoneOffset.UTC).toEpochMilli()', + | if($pv.type == DateTime, + |'ZonedDateTime.parse(params[\'%s\']).toInstant().toEpochMilli()', + | fail('Type not supported on variable - %s:%s'->format([$pv.name, $pv.type->elementToPath()]))->cast(@String) + ) + ), + any: Any[*] | 'params[\'%s\']' + ])->format($param.first); + + let script = '%s.toInstant().toEpochMilli() %s %s'->format([painlessExtractField($tdsESDetail), $operation, $paramScript]); + let literalOrExpression = $newValue->literalOrExpression(true)->toOne(); + pair(^InlineScript(source = $script->literal(), params = newMap(pair($param.first, $literalOrExpression))), $param.second); + }, + {| + let script = '%s %s params[\'%s\']'->format([painlessExtractField($tdsESDetail), $operation, $param.first]); + let literalOrExpression = $value->literalOrExpression(true)->toOne(); + pair(^InlineScript(source = $script->literal(), params = newMap(pair($param.first, $literalOrExpression))), $param.second); + } + ); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessNot(fe: FunctionExpression[1], initReq: State[1]): Pair[1] +{ + let toNot = processPainless($fe.parametersValues->at(0), $initReq); + let toNotScript = $toNot.first; + let script = '!(%s)'->format($toNotScript.source.value->toOne()); + pair(^$toNotScript(source = $script->literal()), $toNot.second); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessAndOr(fe: FunctionExpression[1], initReq: State[1]): Pair[1] +{ + let funcName = $fe.func.functionName->toOne(); + let operation = [ + pair('and', '&&'), + pair('or', '||') + ]->filter(x | $x.first == $funcName).second->toOne('Only support and/or but found: ' + $funcName); + + let left = processPainless($fe.parametersValues->at(0), $initReq); + let right = processPainless($fe.parametersValues->at(1), $left.second); + + let leftScript = $left.first.source.value->toOne(); + let rightScript = $right.first.source.value->toOne(); + + let params = $left.first.params->toOne()->keyValues()->concatenate($right.first.params->toOne()->keyValues())->newMap(); + + let script = '(%s) %s (%s)'->format([$leftScript, $operation, $rightScript]); + pair(^InlineScript(source = $script->literal(), params = $params), $right.second); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessIf(fe: FunctionExpression[1], initReq: State[1]): Pair[1] +{ + let condition = processPainless($fe.parametersValues->at(0), $initReq); + let ifTrue = processPainless($fe.parametersValues->at(1), $condition.second); + let ifFalse = processPainless($fe.parametersValues->at(2), $ifTrue.second); + + let conditionScript = $condition.first.source.value->toOne(); + let ifTrueScript = $ifTrue.first.source.value->toOne(); + let ifFalseScript = $ifFalse.first.source.value->toOne(); + + let params = $condition.first.params->toOne()->keyValues() + ->concatenate($ifTrue.first.params->toOne()->keyValues()) + ->concatenate($ifFalse.first.params->toOne()->keyValues()) + ->newMap(); + + let script = '(%s) ? (%s) : (%s)'->format([$conditionScript, $ifTrueScript, $ifFalseScript]); + pair(^InlineScript(source = $script->literal(), params = $params), $ifFalse.second); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessDateTruncate(fe: FunctionExpression[1], initReq: State[1], unit: String[1]): Pair[1] +{ + let dateFieldPair = $fe.parametersValues->at(0)->extractSimpleValue($initReq); + + let dateField = $dateFieldPair.first.values->cast(@TDSESDetail)->toOne(); + let req = $dateFieldPair.second; + + let script = '%s.truncatedTo(ChronoUnit.%s)'->format([painlessExtractField($dateField), $unit]); + pair(^InlineScript(source = $script->literal(), params = newMap([])), $req); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessExtractDatePart(fe: FunctionExpression[1], initReq: State[1], field: String[1]): Pair[1] +{ + let dateFieldPair = $fe.parametersValues->at(0)->extractSimpleValue($initReq); + + let dateField = $dateFieldPair.first.values->cast(@TDSESDetail)->toOne(); + let req = $dateFieldPair.second; + + let script = '%s.getLong(%s)'->format([painlessExtractField($dateField), $field]); + pair(^InlineScript(source = $script->literal(), params = newMap([])), $req); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessDateFormat(fe: FunctionExpression[1], initReq: State[1], format: String[1]): Pair[1] +{ + let dateFieldPair = $fe.parametersValues->at(0)->extractSimpleValue($initReq); + + let dateField = $dateFieldPair.first.values->cast(@TDSESDetail)->toOne(); + let req = $dateFieldPair.second; + + let param = painlessParameter($req); + + let script = '%s.format(DateTimeFormatter.ofPattern(params[\'%s\']))'->format([painlessExtractField($dateField), $param.first]); + pair(^InlineScript(source = $script->literal(), params = newMap(pair($param.first, $format->literal()))), $param.second); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessFirstDayOfWeek(fe: FunctionExpression[1], initReq: State[1]): Pair[1] +{ + let dateFieldPair = $fe.parametersValues->at(0)->extractSimpleValue($initReq); + + let dateField = $dateFieldPair.first.values->cast(@TDSESDetail)->toOne(); + let req = $dateFieldPair.second; + + let script = '%s.with(DayOfWeek.MONDAY).truncatedTo(ChronoUnit.DAYS)'->format([painlessExtractField($dateField)]); + pair(^InlineScript(source = $script->literal(), params = newMap([])), $req); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessFirstDayOfMonth(fe: FunctionExpression[1], initReq: State[1]): Pair[1] +{ + let dateFieldPair = $fe.parametersValues->at(0)->extractSimpleValue($initReq); + + let dateField = $dateFieldPair.first.values->cast(@TDSESDetail)->toOne(); + let req = $dateFieldPair.second; + + let script = '%s.with(ChronoField.DAY_OF_MONTH, 1).truncatedTo(ChronoUnit.DAYS)'->format(painlessExtractField($dateField)); + pair(^InlineScript(source = $script->literal(), params = newMap([])), $req); +} + +function meta::external::store::elasticsearch::v7::pureToEs::processPainlessFirstDayOfQuarter(fe: FunctionExpression[1], initReq: State[1]): Pair[1] +{ + let dateFieldPair = $fe.parametersValues->at(0)->extractSimpleValue($initReq); + + let dateField = $dateFieldPair.first.values->cast(@TDSESDetail)->toOne(); + let req = $dateFieldPair.second; + + let painlessField = $dateField->painlessExtractField(); + let script = '%s.with(ChronoField.MONTH_OF_YEAR, (%s.get(IsoFields.QUARTER_OF_YEAR) * 3) - 2).with(ChronoField.DAY_OF_MONTH, 1).truncatedTo(ChronoUnit.DAYS)'->format([$painlessField, $painlessField]); + pair(^InlineScript(source = $script->literal(), params = newMap([])), $req); +} + +function meta::external::store::elasticsearch::v7::pureToEs::painlessDateFunctionsFormat(func: Function[1]): String[0..1] +{ + painlessDateSupportedFunctions()->filter(p | $p.first == $func).second.format->first(); +} + +function meta::external::store::elasticsearch::v7::pureToEs::painlessDateSupportedFunctions(): Pair, PainlessDateSupportedFunction>[*] +{ + [ + pair(datePart_Date_1__Date_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessDateTruncate($fe, $req, 'DAYS')}, 'yyyy-MM-dd')) + ,pair(firstMillisecondOfSecond_Date_1__DateTime_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessDateTruncate($fe, $req, 'SECONDS')}, 'yyyy-MM-dd\'T\'HH:mm:ss.SSSZ')) + ,pair(firstSecondOfMinute_Date_1__DateTime_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessDateTruncate($fe, $req, 'MINUTES')}, 'yyyy-MM-dd\'T\'HH:mm:ssZ')) + ,pair(firstMinuteOfHour_Date_1__DateTime_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessDateTruncate($fe, $req, 'HOURS')}, 'yyyy-MM-dd\'T\'HH:mmZ')) + ,pair(firstHourOfDay_Date_1__DateTime_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessDateTruncate($fe, $req, 'DAYS')}, 'yyyy-MM-dd\'T\'HH')) + + ,pair(firstDayOfMonth_Date_1__Date_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessFirstDayOfMonth($fe, $req)}, 'yyyy-MM-dd')) + ,pair(firstDayOfWeek_Date_1__Date_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessFirstDayOfWeek($fe, $req)}, 'yyyy-MM-dd')) + ,pair(firstDayOfQuarter_Date_1__StrictDate_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessFirstDayOfQuarter($fe, $req)}, 'yyyy-MM-dd')) + + ,pair(year_Date_1__Integer_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessExtractDatePart($fe, $req, 'ChronoField.YEAR')}, [])) + ,pair(weekOfYear_Date_1__Integer_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessExtractDatePart($fe, $req, 'IsoFields.WEEK_OF_WEEK_BASED_YEAR')}, [])) + + ,pair(monthNumber_Date_1__Integer_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessExtractDatePart($fe, $req, 'ChronoField.MONTH_OF_YEAR')}, [])) + ,pair(month_Date_1__Month_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessDateFormat($fe, $req, 'MMMM')}, [])) + + ,pair(quarterNumber_Date_1__Integer_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessExtractDatePart($fe, $req, 'IsoFields.QUARTER_OF_YEAR')}, [])) + ,pair(quarter_Date_1__Quarter_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessDateFormat($fe, $req, '\'Q\'q')}, [])) + + ,pair(dayOfWeekNumber_Date_1__Integer_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessExtractDatePart($fe, $req, 'ChronoField.DAY_OF_WEEK')}, [])) + ,pair(dayOfWeek_Date_1__DayOfWeek_1_, + painlessDateSupportedFunction({fe: FunctionExpression[1], req: State[1] | processPainlessDateFormat($fe, $req, 'EEEE')}, [])) + ] +} + +function meta::external::store::elasticsearch::v7::pureToEs::painlessDateSupportedFunction(func: Function<{FunctionExpression[1],State[1]->Pair[1]}>[1], format: String[0..1]): PainlessDateSupportedFunction[1] +{ + ^PainlessDateSupportedFunction( + processor = $func, + format = $format + ) +} + +Class meta::external::store::elasticsearch::v7::pureToEs::PainlessDateSupportedFunction +{ + processor: Function<{FunctionExpression[1],State[1]->Pair[1]}>[1]; + format: String[0..1]; +} + +function meta::external::store::elasticsearch::v7::pureToEs::supportedForPainlessScriptFunctions(): Pair[1],State[1]->Boolean[1]}>, Function<{FunctionExpression[1],State[1]->Pair[1]}>>[*] +{ + let supported = [ + pair(supportedIfEqual(extractEnumValue_Enumeration_1__String_1__T_1_), processPainlessEnumValue_FunctionExpression_1__State_1__Pair_1_) + + ,pair(supportedIfEqual(isEmpty_Any_$0_1$__Boolean_1_), processPainlessIsEmpty_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(isNotEmpty_Any_$0_1$__Boolean_1_), processPainlessIsNotEmpty_FunctionExpression_1__State_1__Pair_1_) + + ,pair({x: Function[1], req: State[1] | $x->instanceOf(QualifiedProperty) && $x->cast(@QualifiedProperty).name == 'isNull'}, + processPainlessIsEmpty_FunctionExpression_1__State_1__Pair_1_) + ,pair({x: Function[1], req: State[1] | $x->instanceOf(QualifiedProperty) && $x->cast(@QualifiedProperty).name == 'isNotNull'}, + processPainlessIsNotEmpty_FunctionExpression_1__State_1__Pair_1_) + + + ,pair(supportedIfEqual(equal_Any_MANY__Any_MANY__Boolean_1_), processPainlessBooleanComparison_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(not_Boolean_1__Boolean_1_), processPainlessNot_FunctionExpression_1__State_1__Pair_1_) + + ,pair(supportedIfEqual(or_Boolean_1__Boolean_1__Boolean_1_), processPainlessAndOr_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(and_Boolean_1__Boolean_1__Boolean_1_), processPainlessAndOr_FunctionExpression_1__State_1__Pair_1_) + + ,pair(supportedIfEqual(greaterThan_Number_1__Number_1__Boolean_1_), processPainlessBooleanComparison_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(greaterThanEqual_Number_1__Number_1__Boolean_1_), processPainlessBooleanComparison_FunctionExpression_1__State_1__Pair_1_) + + ,pair(supportedIfEqual(greaterThan_Date_1__Date_1__Boolean_1_), processPainlessBooleanComparison_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(greaterThanEqual_Date_1__Date_1__Boolean_1_), processPainlessBooleanComparison_FunctionExpression_1__State_1__Pair_1_) + + ,pair(supportedIfEqual(lessThan_Number_1__Number_1__Boolean_1_), processPainlessBooleanComparison_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(lessThanEqual_Number_1__Number_1__Boolean_1_), processPainlessBooleanComparison_FunctionExpression_1__State_1__Pair_1_) + + ,pair(supportedIfEqual(lessThan_Date_1__Date_1__Boolean_1_), processPainlessBooleanComparison_FunctionExpression_1__State_1__Pair_1_) + ,pair(supportedIfEqual(lessThanEqual_Date_1__Date_1__Boolean_1_), processPainlessBooleanComparison_FunctionExpression_1__State_1__Pair_1_) + + ,pair(supportedIfEqual(if_Boolean_1__Function_1__Function_1__T_m_), processPainlessIf_FunctionExpression_1__State_1__Pair_1_) + + ]->concatenate( + painlessDateSupportedFunctions()->map(x | pair(supportedIfEqual($x.first), $x.second.processor)) + ); } \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/specification_utils.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/specification_utils.pure index 1399ba01a2e..04e1782db40 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/specification_utils.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/specification_utils.pure @@ -199,4 +199,24 @@ function <> meta::external::store::elasticsearch::v7::specification:: ]; assertJsonStringsEqual($expected->meta::json::toJSON(), $actual->meta::json::toJSON()); +} + +function meta::external::store::elasticsearch::v7::specification::utils::property::defaultRuntimePropertyForPureType(type: DataType[1]): PropertyBase[1] +{ + if ($type == String || $type->_subTypeOf(Enum), + | ^KeywordProperty(), + | if ($type == Integer, + | ^LongNumberProperty(), + | if ($type == Float, + | ^DoubleNumberProperty(), + | if ($type == Boolean, + | ^BooleanProperty(), + | if ($type == Date || $type == DateTime || $type == StrictDate, + | ^DateProperty(), + | fail(|'Datatype not supported for runtime mapping: ' + $type->elementToPath())->cast(@PropertyBase) + ) + ) + ) + ) + ) } \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/shared/ElasticsearchCommands.java b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/shared/ElasticsearchCommands.java index 843cf393a22..893eb6fa3bf 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/shared/ElasticsearchCommands.java +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/shared/ElasticsearchCommands.java @@ -74,7 +74,9 @@ private static ElasticsearchContainer createContainer(String imageTag) ElasticsearchContainer container = new ElasticsearchContainer(image); long start = System.currentTimeMillis(); - container.withPassword(getPassword()).start(); + container.withPassword(getPassword()) + .withEnv("xpack.security.enabled", "false") + .start(); LOGGER.info("ES Test cluster for version {} running on {}. Took {}ms to start.", imageTag, container.getHttpHostAddress(), System.currentTimeMillis() - start); return container; } @@ -120,6 +122,7 @@ public static String request(String imageTag, String json) switch (imageTag.charAt(0)) { case '7': + case '8': return requestV7(url, json); default: throw new RuntimeException("Version not supported yet: " + imageTag); diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test.pure index fc0ce310975..c9910fe619f 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test.pure @@ -93,7 +93,7 @@ function meta::external::store::elasticsearch::executionTest::testCase::omdbComm Title = $title, Director = $director, MPAA = $MPAA, - ReleaseDate = format('%t{yyyy-MM-dd hh:mm:ssa}', $releaseDate), + ReleaseDate = format('%t{yyyy-MM-dd hh:mm:ss.SSSa}', $releaseDate), ReleaseDateRaw = $releaseDate, Budget = $budget, Revenue = $revenue, @@ -106,47 +106,47 @@ function meta::external::store::elasticsearch::executionTest::testCase::omdbComm { // at the moment,dates are hard to distinguish between StrictDate or DateTime, hence we are testing using DateTime for now let records = [ - omdbCommonIndexRecord('Iron Man', 'Jon Favreau', 'PG-13', %2008-05-02T00:00:00, 140, 585.8, + omdbCommonIndexRecord('Iron Man', 'Jon Favreau', 'PG-13', %2008-05-02T00:00:00.100, 140, 585.8, 'Tells the story of Tony Stark, a billionaire industrialist and genius inventor who is kidnapped and forced to build a devastating weapon', 'Robert Downey Jr.' ), - omdbCommonIndexRecord('Iron Man 2', 'Jon Favreau', 'PG-13', %2010-04-26T00:00:00, 170, 623.9, + omdbCommonIndexRecord('Iron Man 2', 'Jon Favreau', 'PG-13', %2010-04-26T00:00:00.200, 170, 623.9, 'With the world now aware of his identity as Iron Man, Tony Stark must contend with both his declining health and a vengeful mad man with ties to his father\'s legacy.', 'Robert Downey Jr.' ), - omdbCommonIndexRecord('The Imitation Game', 'Morten Tyldum', 'PG-13', %2014-08-29T00:00:00, 14, 233.5, + omdbCommonIndexRecord('The Imitation Game', 'Morten Tyldum', 'PG-13', %2014-08-29T00:00:00.300, 14, 233.5, 'In 1951, two policemen, Nock and Staehl, investigate the mathematician Alan Turing after an apparent break-in at his home', 'Benedict Cumberbatch' ), - omdbCommonIndexRecord('The Bourne Identity', 'Doug Liman', 'PG-13', %2002-06-06T00:00:00, 60, 214.0, + omdbCommonIndexRecord('The Bourne Identity', 'Doug Liman', 'PG-13', %2002-06-06T00:00:00.400, 60, 214.0, 'Jason Bourne, a man suffering from psychogenic amnesia attempting to discover his identity amidst a clandestine conspiracy within the CIA', 'Matt Damon' ), - omdbCommonIndexRecord('American Gangster', 'Ridley Scott', 'R', %2007-10-19T00:00:00, 100, 266.5, + omdbCommonIndexRecord('American Gangster', 'Ridley Scott', 'R', %2007-10-19T00:00:00.500, 100, 266.5, [], 'Denzel Washington' ), - omdbCommonIndexRecord('Sherlock Holmes: A Game of Shadows', 'Guy Ritchie', [], %2011-12-16T00:00:00, 125, 543.8, + omdbCommonIndexRecord('Sherlock Holmes: A Game of Shadows', 'Guy Ritchie', [], %2011-12-16T00:00:00.600, 125, 543.8, 'Sherlock Holmes is investigating a seemingly unrelated series of crimes around Europe, believing them all connected to Professor Moriarty', 'Robert Downey Jr.' ), - omdbCommonIndexRecord('Ghostbusters', 'Ivan Reitman', 'PG', %1984-06-08T00:00:00, 25, 295.2, + omdbCommonIndexRecord('Ghostbusters', 'Ivan Reitman', 'PG', %1984-06-08T00:00:00.700, 25, 295.2, 'Three parapsychologists forced out of their university funding set up shop as a unique ghost removal service in New York City, attracting frightened yet skeptical customers.', 'Bill Murray' ), - omdbCommonIndexRecord('The Hitchhiker\'s Guide to the Galaxy', 'Garth Jennings', [], %2005-04-28T00:00:00, 45, 104.5, + omdbCommonIndexRecord('The Hitchhiker\'s Guide to the Galaxy', 'Garth Jennings', [], %2005-04-28T00:00:00.800, 45, 104.5, 'Mere seconds before the Earth is to be demolished by an alien construction crew, journeyman Arthur Dent is swept off the planet by his friend Ford Prefect, a researcher penning a new edition of "The Hitchhiker\'s Guide to the Galaxy."', 'Martin Freeman' ), - omdbCommonIndexRecord('Inspector Gadget', 'David Kellogg', [], %1999-07-23T00:00:00, 75, 134.4, + omdbCommonIndexRecord('Inspector Gadget', 'David Kellogg', [], %1999-07-23T00:00:00.900, 75, 134.4, 'A security guard\'s dreams come true when he is selected to be transformed into a cybernetic police officer.', 'Don Adams' ), - omdbCommonIndexRecord('The Great Gatsby', 'Baz Luhrmann', [], %2013-05-01T00:00:00, 105, 353.6, + omdbCommonIndexRecord('The Great Gatsby', 'Baz Luhrmann', [], %2013-05-01T00:00:00.000, 105, 353.6, 'The film follows the life and times of millionaire Jay Gatsby and his neighbor Nick Carraway, who recounts his encounter with Gatsby at the height of the Roaring Twenties on Long Island in New York.', 'Leonardo DiCaprio' ), - omdbCommonIndexRecord('Garfield 2', 'Tim Hill', [], %2006-06-16T00:00:00, 60, 143.3, + omdbCommonIndexRecord('Garfield 2', 'Tim Hill', [], %2006-06-16T00:00:01.000, 60, 143.3, 'Garfield follows Jon to England and receives the royal treatment after he is mistaken for the heir to a grand castle.', 'Bill Murray' ) @@ -242,6 +242,8 @@ function meta::external::store::elasticsearch::executionTest::testCase::tds::run function meta::external::store::elasticsearch::executionTest::testCase::execute(config: TestConfig[1], f: FunctionDefinition[1], vars: Pair[*]): TabularDataSet[1] { + // $f->printFunctionDefinition(' ')->println(); + let resultJson = meta::legend::execute( $f, $vars, @@ -277,14 +279,14 @@ function meta::external::store::elasticsearch::executionTest::testCase::assertTd [ $expected.columns.name->joinStrings('[', ', ', ']'), $actual.columns.name->joinStrings('[', ', ', ']') ] ); - // println($expected.rows.values); - // println($actual.rows.values); - assertEquals($expected.rows->size(), $actual.rows->size(), 'Number of rows are different. Expected %d, got %d', [ $expected.rows->size(), $actual.rows->size() ] ); + // println($expected.rows.values); + // println($actual.rows.values); + let zipped = if ($description->contains('sort('), | $expected.rows->map(r | $expected.columns.name->map(c | $r.get($c))) ->zip($actual.rows->map(r | $actual.columns.name->map(c | $r.get($c)))), @@ -292,13 +294,17 @@ function meta::external::store::elasticsearch::executionTest::testCase::assertTd ->zip($actual->sort($actual.columns.name).rows->map(r | $actual.columns.name->map(c | $r.get($c)))) ); + // $zipped->map(p | $p.first->toString() + ' == ' + $p.second->toString())->println(); + $zipped->forAll(x | assert(comparePrimitives($x.first, $x.second))); } function <> meta::external::store::elasticsearch::executionTest::testCase::comparePrimitives(firstInstance: Any[1], secondInstance: Any[1]): Boolean[1] { $firstInstance->match([ n: Number[1] | assertEqWithinTolerance($n, $secondInstance->cast(@Number), 0.0001), - a: Any[1] | assertEquals($a, $secondInstance)]); + e: Enum[1] | assertEquals($e, extractEnumValue($e->type()->cast(@Enumeration), $secondInstance->cast(@String))), + a: Any[1] | assertEquals($a, $secondInstance) + ]); } function meta::external::store::elasticsearch::executionTest::testCase::asZeroOne(var: T[0..1]): T[0..1] diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_7.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_7.pure index 706a945ad0a..40f3c46203b 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_7.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_7.pure @@ -48,7 +48,7 @@ function <> meta::external::store::elasticsearch::executionTest: ), pair('Director', ^meta::external::store::elasticsearch::v7::metamodel::specification::types::mapping::Property(keyword = ^KeywordProperty())), pair('MPAA', ^meta::external::store::elasticsearch::v7::metamodel::specification::types::mapping::Property(keyword = ^KeywordProperty())), - pair('ReleaseDate', ^meta::external::store::elasticsearch::v7::metamodel::specification::types::mapping::Property(date = ^DateProperty(format = 'yyyy-MM-dd hh:mm:ssa'->literal()))), + pair('ReleaseDate', ^meta::external::store::elasticsearch::v7::metamodel::specification::types::mapping::Property(date = ^DateProperty(format = 'yyyy-MM-dd hh:mm:ss.SSSa'->literal()))), pair('Budget', ^meta::external::store::elasticsearch::v7::metamodel::specification::types::mapping::Property(integer = ^IntegerNumberProperty())), pair('Revenue', ^meta::external::store::elasticsearch::v7::metamodel::specification::types::mapping::Property(_float = ^FloatNumberProperty())), pair('MainActor', ^meta::external::store::elasticsearch::v7::metamodel::specification::types::mapping::Property(object = @@ -77,7 +77,7 @@ function <> meta::external::store::elasticsearch::executionTest: function <> meta::external::store::elasticsearch::executionTest::test::v7::index(name: String[1], records: Any[*]): Any[*] { - let deletIndex = ^DeleteRequest( + let deleteIndex = ^DeleteRequest( index = $name->literal(), ignore_unavailable = true->literal() ); @@ -103,7 +103,7 @@ function <> meta::external::store::elasticsearch::executionTest: ); [ - $deletIndex, + $deleteIndex, $createIndex, $indexData, $countData @@ -121,21 +121,13 @@ function <> meta::external::store::elasticsearch::executionTest: ); } -function <> meta::external::store::elasticsearch::executionTest::test::v7::test_7_8_0(): PureTestCollection[1] +// this test if 7 is fwd compatible for now until we settle on 7 functionality, and create 8 officially... +function <> meta::external::store::elasticsearch::executionTest::test::v7::test_8_6_2(): PureTestCollection[1] { - let testToIgnore = [ - // start multi_term ignore - not supported until 7.12 - meta::external::store::elasticsearch::executionTest::testCase::tds::groupBy::sort::testSortOnSingleAggFields_multipleGroupByFields_TestConfig_1__Boolean_1_, - meta::external::store::elasticsearch::executionTest::testCase::tds::groupBy::sort::testSortOnSingleAggFields_multipleGroupByFields_withRename_TestConfig_1__Boolean_1_, - meta::external::store::elasticsearch::executionTest::testCase::tds::groupBy::sort::testSortOnMultipleAggFields_TestConfig_1__Boolean_1_, - meta::external::store::elasticsearch::executionTest::testCase::tds::groupBy::sort::testSortOnMultipleAggFields_withRename_TestConfig_1__Boolean_1_ - // end multi_term ignore - not supported until 7.12 - ]; - - '7.8.0'->collectTest(x | $x->in($testToIgnore)->not()); + '8.6.2'->collectTest([]); } function <> meta::external::store::elasticsearch::executionTest::test::v7::test_7_17_0(): PureTestCollection[1] { - '7.17.0'->collectTest([]); + '7.17.7'->collectTest([]); } \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_boolean.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_boolean.pure new file mode 100644 index 00000000000..eaba4b2f81b --- /dev/null +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_boolean.pure @@ -0,0 +1,39 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::test::*; +import meta::external::store::elasticsearch::executionTest::testCase::*; +import meta::external::store::elasticsearch::executionTest::testCase::tds::*; +import meta::external::store::elasticsearch::executionTest::test::*; +import meta::external::store::elasticsearch::executionTest::utils::*; + +function + <> +meta::external::store::elasticsearch::executionTest::testCase::tds::filter::boolean::testFilterUsingBooleanIsNotEmptyField(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x + ->extend(col(x: TDSRow[1] | $x.getNullableString('MPAA')->isNotEmpty(), 'missing mpaa')) + ->filter(x | $x.getBoolean('missing mpaa')) + ); +} + +function + <> +meta::external::store::elasticsearch::executionTest::testCase::tds::filter::boolean::testFilterUsingBooleanIsEmptyField(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x + ->extend(col(x: TDSRow[1] | $x.getNullableString('MPAA')->isEmpty(), 'has mpaa')) + ->filter(x | $x.getBoolean('has mpaa')) + ); +} \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_date.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_date.pure index db32c64f8db..d69e338dd57 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_date.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_date.pure @@ -23,7 +23,7 @@ function {doc.doc = 'Test equal filter on Elasticsearch Keyword property mapping'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testEqualValueFilter(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') == %1984-06-08T00:00:00+0000)); + $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') == %1984-06-08T00:00:00.700+0000)); } function @@ -31,7 +31,7 @@ function {doc.doc = 'Test equal filter on Elasticsearch Keyword property mapping'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testNotEqualValueFilter(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') != %1984-06-08T00:00:00+0000)); + $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') != %1984-06-08T00:00:00.700+0000)); } function @@ -63,7 +63,7 @@ function {doc.doc = 'Test greater than equal filter on Elasticsearch Integer property mapping'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testGreaterThanEqualReversedDate(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->filter(x | %2010-04-26T00:00:00+0000 >= $x.getDate('ReleaseDate'))); + $config->testTdsExpression(x | $x->filter(x | %2010-04-26T00:00:00.200+0000 >= $x.getDate('ReleaseDate'))); } function @@ -71,7 +71,7 @@ function {doc.doc = 'Test less than filter on Elasticsearch Integer property mapping'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testLessThanDate(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') < %2010-04-26T00:00:00+0000)); + $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') < %2010-04-26T00:00:00.200+0000)); } function @@ -79,7 +79,7 @@ function {doc.doc = 'Test less than filter on Elasticsearch Integer property mapping'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testLessThanReversedDate(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->filter(x | %2010-04-26T00:00:00+0000 < $x.getDate('ReleaseDate'))); + $config->testTdsExpression(x | $x->filter(x | %2010-04-26T00:00:00.200+0000 < $x.getDate('ReleaseDate'))); } function @@ -87,7 +87,7 @@ function {doc.doc = 'Test less than equal filter on Elasticsearch Integer property mapping'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testLessThanEqualDate(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') <= %2010-04-26T00:00:00+0000)); + $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') <= %2010-04-26T00:00:00.200+0000)); } function @@ -95,7 +95,7 @@ function {doc.doc = 'Test less than equal filter on Elasticsearch Integer property mapping'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testLessThanEqualReversedDate(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->filter(x | %2010-04-26T00:00:00+0000 <= $x.getDate('ReleaseDate'))); + $config->testTdsExpression(x | $x->filter(x | %2010-04-26T00:00:00.200+0000 <= $x.getDate('ReleaseDate'))); } function @@ -103,7 +103,7 @@ function {doc.doc = 'Test equal filter on Elasticsearch Keyword property mapping with variable'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testEqualValueFilter_PureOneVariable(config:TestConfig[1]):Boolean[1] { - let var = %1984-06-08T00:00:00+0000; + let var = %1984-06-08T00:00:00.700+0000; $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') == $var)); } @@ -112,7 +112,7 @@ function {doc.doc = 'Test equal filter on Elasticsearch Keyword property mapping with variable'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testNotEqualValueFilter_PureOneVariable(config:TestConfig[1]):Boolean[1] { - let var = %1984-06-08T00:00:00+0000; + let var = %1984-06-08T00:00:00.700+0000; $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') != $var)); } @@ -139,7 +139,7 @@ function {doc.doc = 'Test greater than equal filter on Elasticsearch Integer property mapping with variable'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testGreaterThanEqualDate_PureOneVariable(config:TestConfig[1]):Boolean[1] { - let var = %1984-06-08T00:00:00+0000; + let var = %1984-06-08T00:00:00.700+0000; $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') >= $var)); } @@ -148,7 +148,7 @@ function {doc.doc = 'Test greater than equal filter on Elasticsearch Integer property with variable mapping'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testGreaterThanEqualReversedDate_PureOneVariable(config:TestConfig[1]):Boolean[1] { - let var = %2010-04-26T00:00:00+0000; + let var = %2010-04-26T00:00:00.200+0000; $config->testTdsExpression(x | $x->filter(x | $var >= $x.getDate('ReleaseDate'))); } @@ -157,7 +157,7 @@ function {doc.doc = 'Test less than filter on Elasticsearch Integer property mapping with variable'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testLessThanDate_PureOneVariable(config:TestConfig[1]):Boolean[1] { - let var = %2010-04-26T00:00:00+0000; + let var = %2010-04-26T00:00:00.200+0000; $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') < $var)); } @@ -166,7 +166,7 @@ function {doc.doc = 'Test less than filter on Elasticsearch Integer property mapping with variable'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testLessThanReversedDate_PureOneVariable(config:TestConfig[1]):Boolean[1] { - let var = %2010-04-26T00:00:00+0000; + let var = %2010-04-26T00:00:00.200+0000; $config->testTdsExpression(x | $x->filter(x | $var < $x.getDate('ReleaseDate'))); } @@ -175,7 +175,7 @@ function {doc.doc = 'Test less than equal filter on Elasticsearch Integer property mapping with variable'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testLessThanEqualDate_PureOneVariable(config:TestConfig[1]):Boolean[1] { - let var = %2010-04-26T00:00:00+0000; + let var = %2010-04-26T00:00:00.200+0000; $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') <= $var)); } @@ -184,6 +184,6 @@ function {doc.doc = 'Test less than equal filter on Elasticsearch Integer property mapping with variable'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::date::testLessThanEqualReversedDate_PureOneVariable(config:TestConfig[1]):Boolean[1] { - let var = %2010-04-26T00:00:00+0000; + let var = %2010-04-26T00:00:00.200+0000; $config->testTdsExpression(x | $x->filter(x | $var <= $x.getDate('ReleaseDate'))); } \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_enum.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_enum.pure new file mode 100644 index 00000000000..1422ec8c883 --- /dev/null +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_enum.pure @@ -0,0 +1,31 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::test::*; +import meta::external::store::elasticsearch::executionTest::testCase::*; +import meta::external::store::elasticsearch::executionTest::testCase::tds::*; +import meta::external::store::elasticsearch::executionTest::test::*; +import meta::external::store::elasticsearch::executionTest::utils::*; + + +function + <> + +meta::external::store::elasticsearch::executionTest::testCase::tds::filter::enum::testFilterEnum(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x + ->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->dayOfWeek(), 'dayOfWeek')) + ->filter(x | $x.getEnum('dayOfWeek') == DayOfWeek.Monday) + ); +} \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_expressions.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_expressions.pure index 3ee5fa5f6e3..9f34ed60562 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_expressions.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_filter_expressions.pure @@ -79,7 +79,7 @@ function {doc.doc = 'Test filter date adjust expression'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::expressions::testAdjustExpression(config:TestConfig[1]):Boolean[1] { - let val = %1984-06-07T00:00:00+0000; + let val = %1984-06-07T00:00:00.700+0000; $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') == adjust($val, 1, DurationUnit.DAYS))); } @@ -104,6 +104,6 @@ function {doc.doc = 'Test filter adjust today expression'} meta::external::store::elasticsearch::executionTest::testCase::tds::filter::expressions::testAdjustTodayExpression(config:TestConfig[1]):Boolean[1] { - let val = -1 * dateDiff(%1984-06-08T00:00:00+0000, now(), DurationUnit.DAYS); - $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') == adjust(parseDate(today()->toString() + 'T00:00:00'), $val, DurationUnit.DAYS))); + let val = -1 * dateDiff(%1984-06-08T00:00:00.700+0000, now(), DurationUnit.DAYS); + $config->testTdsExpression(x | $x->filter(x | $x.getDate('ReleaseDate') == adjust(parseDate(today()->toString() + 'T00:00:00.700'), $val, DurationUnit.DAYS))); } \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_misc.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_misc.pure index 31c15eb79ff..7257c358512 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_misc.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_misc.pure @@ -23,7 +23,7 @@ function {doc.doc = 'Test limit on Elasticsearch'} meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testLimit(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->sort('_id')->limit(3)); + $config->testTdsExpression(x | $x->sort('Title')->limit(3)); } function @@ -31,32 +31,32 @@ function {doc.doc = 'Test take on Elasticsearch'} meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testTake(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->sort('_id')->take(3)); + $config->testTdsExpression(x | $x->sort('Title')->take(3)); } -function +function <> - {doc.doc = 'Test drop on Elasticsearch'} + {doc.doc = 'Test drop on Elasticsearch'} meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testDrop(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->sort('_id')->drop(3)); + $config->testTdsExpression(x | $x->sort('Title')->drop(3)); } -function +function <> - {doc.doc = 'Test drop and limit together on Elasticsearch'} + {doc.doc = 'Test drop and limit together on Elasticsearch'} meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testDropAndLimit(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->sort('_id')->drop(3)->limit(3)); + $config->testTdsExpression(x | $x->sort('Title')->drop(3)->limit(3)); } -function +function <> - {doc.doc = 'Test limit and drop together on Elasticsearch'} + {doc.doc = 'Test limit and drop together on Elasticsearch'} meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testLimitAndDrop(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->sort('_id')->limit(4)->drop(3)); - $config->testTdsExpressionReturnsEmpty(x | $x->sort('_id')->limit(3)->drop(3)); + $config->testTdsExpression(x | $x->sort('Title')->limit(4)->drop(3)); + $config->testTdsExpressionReturnsEmpty(x | $x->sort('Title')->limit(3)->drop(3)); } function @@ -64,8 +64,8 @@ function {doc.doc = 'Test slice on Elasticsearch'} meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testSlice(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x | $x->sort('_id')->slice(0, 1)); - $config->testTdsExpression(x | $x->sort('_id')->slice(3, 4)); + $config->testTdsExpression(x | $x->sort('Title')->slice(0, 1)); + $config->testTdsExpression(x | $x->sort('Title')->slice(3, 4)); } function @@ -74,7 +74,7 @@ function meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testLimit_PureOneVariable(config:TestConfig[1]):Boolean[1] { let val = 3; - $config->testTdsExpression(x | $x->sort('_id')->limit($val)); + $config->testTdsExpression(x | $x->sort('Title')->limit($val)); } function @@ -83,7 +83,7 @@ function meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testTake_PureOneVariable(config:TestConfig[1]):Boolean[1] { let val = 3; - $config->testTdsExpression(x | $x->sort('_id')->take($val)); + $config->testTdsExpression(x | $x->sort('Title')->take($val)); } function @@ -92,7 +92,7 @@ function meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testDrop_PureOneVariable(config:TestConfig[1]):Boolean[1] { let val = 3; - $config->testTdsExpression(x | $x->sort('_id')->drop($val)); + $config->testTdsExpression(x | $x->sort('Title')->drop($val)); } function @@ -102,34 +102,34 @@ meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testDr { let var = 3; - $config->testTdsExpression(x | $x->sort('_id')->drop($var)->limit($var)); + $config->testTdsExpression(x | $x->sort('Title')->drop($var)->limit($var)); - $config->testTdsExpression(x | $x->sort('_id')->drop(3)->limit($var)); + $config->testTdsExpression(x | $x->sort('Title')->drop(3)->limit($var)); - $config->testTdsExpression(x | $x->sort('_id')->drop($var)->limit(3)); + $config->testTdsExpression(x | $x->sort('Title')->drop($var)->limit(3)); } -function +function <> - {doc.doc = 'Test limit and drop together on Elasticsearch with variable'} + {doc.doc = 'Test limit and drop together on Elasticsearch with variable'} meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testLimitAndDrop_PureOneVariable(config:TestConfig[1]):Boolean[1] { let var = 4; let var2 = 3; - $config->testTdsExpression(x | $x->sort('_id')->limit($var)->drop($var2)); - $config->testTdsExpressionReturnsEmpty(x | $x->sort('_id')->limit($var2)->drop($var2)); + $config->testTdsExpression(x | $x->sort('Title')->limit($var)->drop($var2)); + $config->testTdsExpressionReturnsEmpty(x | $x->sort('Title')->limit($var2)->drop($var2)); - $config->testTdsExpression(x | $x->sort('_id')->limit(4)->drop($var2)); - $config->testTdsExpressionReturnsEmpty(x | $x->sort('_id')->limit(3)->drop($var2)); + $config->testTdsExpression(x | $x->sort('Title')->limit(4)->drop($var2)); + $config->testTdsExpressionReturnsEmpty(x | $x->sort('Title')->limit(3)->drop($var2)); - $config->testTdsExpression(x | $x->sort('_id')->limit($var)->drop(3)); - $config->testTdsExpressionReturnsEmpty(x | $x->sort('_id')->limit($var2)->drop(3)); + $config->testTdsExpression(x | $x->sort('Title')->limit($var)->drop(3)); + $config->testTdsExpressionReturnsEmpty(x | $x->sort('Title')->limit($var2)->drop(3)); } -function +function <> - {doc.doc = 'Test slice on Elasticsearch with variable'} + {doc.doc = 'Test slice on Elasticsearch with variable'} meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testSlice_PureOneVariable(config:TestConfig[1]):Boolean[1] { let var0 = 0; @@ -138,11 +138,11 @@ meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testSl let var3 = 3; let var4 = 4; - $config->testTdsExpression(x | $x->sort('_id')->slice($var0, $var1)); - $config->testTdsExpression(x | $x->sort('_id')->slice(0, $var1)); - $config->testTdsExpression(x | $x->sort('_id')->slice($var0, 1)); + $config->testTdsExpression(x | $x->sort('Title')->slice($var0, $var1)); + $config->testTdsExpression(x | $x->sort('Title')->slice(0, $var1)); + $config->testTdsExpression(x | $x->sort('Title')->slice($var0, 1)); - $config->testTdsExpression(x | $x->sort('_id')->slice($var3, $var4)); - $config->testTdsExpression(x | $x->sort('_id')->slice(3, $var4)); - $config->testTdsExpression(x | $x->sort('_id')->slice($var3, 4)); + $config->testTdsExpression(x | $x->sort('Title')->slice($var3, $var4)); + $config->testTdsExpression(x | $x->sort('Title')->slice(3, $var4)); + $config->testTdsExpression(x | $x->sort('Title')->slice($var3, 4)); } \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure index 2d221e9c422..201f78cf3cd 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure @@ -18,36 +18,70 @@ import meta::external::store::elasticsearch::executionTest::testCase::tds::*; import meta::external::store::elasticsearch::executionTest::test::*; import meta::external::store::elasticsearch::executionTest::utils::*; +// function +// <> +// {doc.doc = 'Test projection on Elasticsearch with pure constant expressions'} +// meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionConstant(config:TestConfig[1]):Boolean[1] +// { +// $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | 'Hello World!', 'Bucket')])); +// } + +// function +// <> +// {doc.doc = 'Test projection on Elasticsearch with pure constant expressions'} +// meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionConstantWithVariable(config:TestConfig[1]):Boolean[1] +// { +// let val = 12345; +// $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $val, 'Bucket')])); +// } function <> - {doc.doc = 'Test projection on Elasticsearch Keyword property mapping'} -meta::external::store::elasticsearch::executionTest::testCase::tds::project::testProjectKeyword(config:TestConfig[1]):Boolean[1] + {doc.doc = 'Test projection on Elasticsearch with pure if expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionIf(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $x.getString('Title'), 'Hello')])); + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | if($x.getDate('ReleaseDate') >= %1990-01-01, |'90s and newer', |'Older than 90s'), 'Bucket')])); } - function <> - {doc.doc = 'Test projection on Elasticsearch Integer property mapping'} -meta::external::store::elasticsearch::executionTest::testCase::tds::project::testProjectInteger(config:TestConfig[1]):Boolean[1] + {doc.doc = 'Test projection on Elasticsearch with pure if expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionIfWithVariables(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $x.getInteger('Budget'), 'Hello')])); + let var = %1990-01-01; + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | if($x.getDate('ReleaseDate') >= $var, |'90s and newer', |'Older than 90s'), 'Bucket')])); } function <> - {doc.doc = 'Test projection on Elasticsearch Float property mapping'} -meta::external::store::elasticsearch::executionTest::testCase::tds::project::testProjectFloat(config:TestConfig[1]):Boolean[1] + {doc.doc = 'Test projection on Elasticsearch with pure nested if expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionNestedIf(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $x.getFloat('Revenue'), 'Hello')])); + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | + if($x.getDate('ReleaseDate') < %1990-01-01, + |'Oldest', + |if($x.getDate('ReleaseDate') < %2000-01-01, + |'Older', + |if($x.getDate('ReleaseDate') < %2010-01-01, + |'Newer', + |'Newest'))), 'Bucket')])); } function <> - {doc.doc = 'Test projection on Elasticsearch Date property mapping'} -meta::external::store::elasticsearch::executionTest::testCase::tds::project::testProjectDate(config:TestConfig[1]):Boolean[1] + {doc.doc = 'Test projection on Elasticsearch with pure nested if expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionNestedIfWithVariables(config:TestConfig[1]):Boolean[1] { - $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $x.getDate('ReleaseDate'), 'Hello')])); + let var1 = %1990-01-01; + let var2 = %2000-01-01; + let var3 = %2010-01-01; + + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | + if($x.getDate('ReleaseDate') < $var1, + |'Oldest', + |if($x.getDate('ReleaseDate') < $var2, + |'Older', + |if($x.getDate('ReleaseDate') < $var3, + |'Newer', + |'Newest'))), 'Bucket')])); } \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_boolean.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_boolean.pure new file mode 100644 index 00000000000..f2ec3195bd2 --- /dev/null +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_boolean.pure @@ -0,0 +1,87 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::test::*; +import meta::external::store::elasticsearch::executionTest::testCase::*; +import meta::external::store::elasticsearch::executionTest::testCase::tds::*; +import meta::external::store::elasticsearch::executionTest::test::*; +import meta::external::store::elasticsearch::executionTest::utils::*; + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure isEmpty expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::boolean::testProjectExpressionIsEmpty(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | $x.getNullableString('MPAA')->isEmpty(), 'has mpaa')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure TdsisNull expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::boolean::testProjectExpressionTdsIsNull(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | $x.isNull('MPAA'), 'has mpaa')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure isNotEmpty expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::boolean::testProjectExpressionIsNotEmpty(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | $x.getNullableString('MPAA')->isNotEmpty(), 'missing mpaa')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure TdsisNotNull expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::boolean::testProjectExpressionTdsIsNotEmpty(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | $x.isNotNull('MPAA'), 'missing mpaa')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure && expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::boolean::testProjectExpressionAnd(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | ($x.getDate('ReleaseDate') >= %1990-01-01) && ($x.getDate('ReleaseDate') < %2000-01-01), '90s')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure && expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::boolean::testProjectExpressionAndWithVariables(config:TestConfig[1]):Boolean[1] +{ + let var1 = %1990-01-01; + let var2 = %2000-01-01; + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | ($x.getDate('ReleaseDate') >= $var1) && ($x.getDate('ReleaseDate') < $var2), '90s')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with || expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::boolean::testProjectExpressionOr(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | ($x.getDate('ReleaseDate') < %1990-01-01) || ($x.getDate('ReleaseDate') >= %2000-01-01), 'Not 90s')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure || expressions'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::boolean::testProjectExpressionOrWithVariables(config:TestConfig[1]):Boolean[1] +{ + let var1 = %1990-01-01; + let var2 = %2000-01-01; + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | ($x.getDate('ReleaseDate') < $var1) && ($x.getDate('ReleaseDate') >= $var2), 'Not 90s')])); +} \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_date.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_date.pure new file mode 100644 index 00000000000..d14eb85760e --- /dev/null +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_date.pure @@ -0,0 +1,238 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::test::*; +import meta::external::store::elasticsearch::executionTest::testCase::*; +import meta::external::store::elasticsearch::executionTest::testCase::tds::*; +import meta::external::store::elasticsearch::executionTest::test::*; +import meta::external::store::elasticsearch::executionTest::utils::*; + +function + <> + {doc.doc = 'Test projection on Elasticsearch Date property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProjectDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $x.getDate('ReleaseDate'), 'RD')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure == expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testEqualValue(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate') == %1984-06-08T00:00:00.700+0000, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure != expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testNotEqualValue(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate') != %1984-06-08T00:00:00.700+0000, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testGreaterThanDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate') > %1985-01-01T00:00:00+0000, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testGreaterThanReversedDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | %2000-01-01T00:00:00+0000 > $x.getDate('ReleaseDate'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testGreaterThanEqualDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate') >= %1984-06-08T00:00:00+0000, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testGreaterThanEqualReversedDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | %2010-04-26T00:00:00.200+0000 >= $x.getDate('ReleaseDate'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testLessThanDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate') < %2010-04-26T00:00:00.200+0000, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testLessThanReversedDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | %2010-04-26T00:00:00.200+0000 < $x.getDate('ReleaseDate'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testLessThanEqualDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate') <= %2010-04-26T00:00:00.200+0000, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testLessThanEqualReversedDate(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | %2010-04-26T00:00:00.200+0000 <= $x.getDate('ReleaseDate'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure datePart expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProjectDatePart(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->datePart(), 'datePart'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure firstMillisecondOfSecond expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProjectFirstMillisecondOfSecond(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->firstMillisecondOfSecond(), 'firstMillisecondOfSecond'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure firstSecondOfMinute expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProjectFirstSecondOfMinute(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->firstSecondOfMinute(), 'firstSecondOfMinute'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure firstMinuteOfHour expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProjectFirstMinuteOfHour(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->firstMinuteOfHour(), 'firstMinuteOfHour'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure firstHourOfDay expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProjectFirstHourOfDay(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->firstHourOfDay(), 'firstHourOfDay'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure firstDayOfMonth expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProjectFirstDayOfMonth(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->firstDayOfMonth(), 'firstDayOfMonth'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure firstDayOfWeek expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProjectFirstDayOfWeek(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->firstDayOfWeek(), 'firstDayOfWeek'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure firstDayOfQuarter expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProjectFirstDayOfQuarter(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->firstDayOfQuarter(), 'firstDayOfQuarter'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure year expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProject_year(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->year(), 'year'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure weekOfYear expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProject_weekOfYear(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->weekOfYear(), 'weekOfYear')) + // todo some dates provide discrepancies, cause by Java calendar poor implementation of week of year, so narrow to values that work + ->filter(x | $x.getDate('ReleaseDate') == %2008-05-02T00:00:00.100) + ); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure monthNumber expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProject_monthNumber(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->monthNumber(), 'monthNumber'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure month expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProject_month(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->month(), 'month'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure quarterNumber expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProject_quarterNumber(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->quarterNumber(), 'quarterNumber'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure quarter expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProject_quarter(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->quarter(), 'quarter'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure dayOfWeekNumber expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProject_dayOfWeekNumber(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->dayOfWeekNumber(), 'dayOfWeekNumber'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure dayOfWeek expression on date'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::date::testProject_dayOfWeek(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend(col(x: TDSRow[1] | $x.getDate('ReleaseDate')->dayOfWeek(), 'dayOfWeek'))); +} \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_float.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_float.pure new file mode 100644 index 00000000000..34563842ef7 --- /dev/null +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_float.pure @@ -0,0 +1,198 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::test::*; +import meta::external::store::elasticsearch::executionTest::testCase::*; +import meta::external::store::elasticsearch::executionTest::testCase::tds::*; +import meta::external::store::elasticsearch::executionTest::test::*; +import meta::external::store::elasticsearch::executionTest::utils::*; + +function + <> + {doc.doc = 'Test projection on Elasticsearch Float property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testProjectFloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $x.getFloat('Revenue'), 'rev')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure == expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testEqualValueProject(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') == 233.5, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure != expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testNotEqualValueProject(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') != 233.5, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testGreaterThanfloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') > 266.5, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testGreaterThanReversedfloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | 266.5 > $x.getFloat('Revenue'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testGreaterThanEqualfloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') >= 266.5, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testGreaterThanEqualReversedfloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | 266.5 >= $x.getFloat('Revenue'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testLessThanfloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') < 266.5, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testLessThanReversedfloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | 266.5 < $x.getFloat('Revenue'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testLessThanEqualfloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') <= 266.5, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testLessThanEqualReversedfloat(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | 266.5 <= $x.getFloat('Revenue'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure == expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testEqualValueFilterWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 233.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') == $var, 'expression'))); +} + + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure != expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testNotEqualValueFilterWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 233.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') != $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testGreaterThanfloatWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 266.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') > $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testGreaterThanReversedfloatWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 266.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $var > $x.getFloat('Revenue'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testGreaterThanEqualfloatWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 266.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') >= $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testGreaterThanEqualReversedfloatWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 266.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $var >= $x.getFloat('Revenue'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testLessThanfloatWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 266.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') < $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testLessThanReversedfloatWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 266.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $var < $x.getFloat('Revenue'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testLessThanEqualfloatWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 266.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getFloat('Revenue') <= $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on float'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::float::testLessThanEqualReversedfloatWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 266.5; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $var <= $x.getFloat('Revenue'), 'expression'))); +} diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_integer.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_integer.pure new file mode 100644 index 00000000000..058c6dc38fe --- /dev/null +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_integer.pure @@ -0,0 +1,190 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::test::*; +import meta::external::store::elasticsearch::executionTest::testCase::*; +import meta::external::store::elasticsearch::executionTest::testCase::tds::*; +import meta::external::store::elasticsearch::executionTest::test::*; +import meta::external::store::elasticsearch::executionTest::utils::*; + +function + <> + {doc.doc = 'Test projection on Elasticsearch Integer property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testProjectInteger(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $x.getInteger('Budget'), 'Hello')])); +} + + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure != expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testNotEqualValue(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') != 170, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testGreaterThan(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') > 55, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testGreaterThanReversedInteger(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | 55 > $x.getInteger('Budget'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testGreaterThanEqualInteger(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') >= 55, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testGreaterThanEqualReversedInteger(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | 55 >= $x.getInteger('Budget'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testLessThanInteger(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') < 55, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testLessThanReversedInteger(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | 55 < $x.getInteger('Budget'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testLessThanEqualInteger(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') <= 55, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testLessThanEqualReversedInteger(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | 55 <= $x.getInteger('Budget'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure == expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testEqualValueFilterWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 170; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') == $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure != expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testNotEqualValueFilterWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 170; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') != $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testGreaterThanIntegerWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 55; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') > $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure > expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testGreaterThanReversedIntegerWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 55; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $var > $x.getInteger('Budget'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testGreaterThanEqualIntegerWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 55; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') >= $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure >= expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testGreaterThanEqualReversedIntegerWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 55; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $var >= $x.getInteger('Budget'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testLessThanInteger_PurOneVariable(config:TestConfig[1]):Boolean[1] +{ + let val = 55; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') < $val, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure < expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testLessThanReversedIntegerWithVariable(config:TestConfig[1]):Boolean[1] +{ + let val = 55; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $val < $x.getInteger('Budget'), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testLessThanEqualIntegerWithVariable(config:TestConfig[1]):Boolean[1] +{ + let val = 55; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getInteger('Budget') <= $val, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure <= expression on integer'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::integer::testLessThanEqualReversedIntegerWithVariable(config:TestConfig[1]):Boolean[1] +{ + let val = 55; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $val <= $x.getInteger('Budget'), 'expression'))); +} diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_keyword.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_keyword.pure new file mode 100644 index 00000000000..16ef0069ea8 --- /dev/null +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_keyword.pure @@ -0,0 +1,61 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::test::*; +import meta::external::store::elasticsearch::executionTest::testCase::*; +import meta::external::store::elasticsearch::executionTest::testCase::tds::*; +import meta::external::store::elasticsearch::executionTest::test::*; +import meta::external::store::elasticsearch::executionTest::utils::*; + +function + <> + {doc.doc = 'Test projection on Elasticsearch Keyword property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::keyword::testProjectKeyword(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->project([col(x: TDSRow[1] | $x.getString('Title'), 'Title')])); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure == expression on keyword'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::keyword::testEqualValue(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getString('Title') == 'Iron Man', 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure != expression on keyword'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::keyword::testNotEqualValue(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getString('Title') != 'Iron Man', 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure == expression on keyword'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::keyword::testEqualValueWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 'Iron Man'; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getString('Title') == $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure != expression on keyword'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::keyword::testNotEqualValueWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 'Iron Man'; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getString('Title') != $var, 'expression'))); +} diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_text.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_text.pure new file mode 100644 index 00000000000..2bd4aa98876 --- /dev/null +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project_text.pure @@ -0,0 +1,77 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::test::*; +import meta::external::store::elasticsearch::executionTest::testCase::*; +import meta::external::store::elasticsearch::executionTest::testCase::tds::*; +import meta::external::store::elasticsearch::executionTest::test::*; +import meta::external::store::elasticsearch::executionTest::utils::*; + +function + <> + {doc.doc = 'Test projection on Elasticsearch Text property mapping'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::text::testProjectText(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->project(col(x: TDSRow[1] | $x.getNullableString('Description'), 'Description'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure isEmpty expression on text'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::text::testIsEmpty(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getNullableString('Description')->isEmpty(), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure isNotEmpty expression on text'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::text::testIsNotEmpty(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getNullableString('Description')->isNotEmpty(), 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure == expression on text'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::text::testEqualValue(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getNullableString('Description') == 'A security guard\'s dreams come true when he is selected to be transformed into a cybernetic police officer.', 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure != expression on text'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::text::testNotEqualValue(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getNullableString('Description') != 'A security guard\'s dreams come true when he is selected to be transformed into a cybernetic police officer.', 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure == expression on text'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::text::testEqualValueWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 'A security guard\'s dreams come true when he is selected to be transformed into a cybernetic police officer.'; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getNullableString('Description') == $var, 'expression'))); +} + +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure != expression on text'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::text::testNotEqualValueWithVariable(config:TestConfig[1]):Boolean[1] +{ + let var = 'A security guard\'s dreams come true when he is selected to be transformed into a cybernetic police officer.'; + $config->testTdsExpression(x | $x->extend(col(x: TDSRow[1] | $x.getNullableString('Description') != $var, 'expression'))); +} \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/test/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/TestElasticsearchExecutionPlanFromGrammarIntegration.java b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/test/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/TestElasticsearchExecutionPlanFromGrammarIntegration.java index 742cf8ffa22..02e922294a4 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/test/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/TestElasticsearchExecutionPlanFromGrammarIntegration.java +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/test/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/TestElasticsearchExecutionPlanFromGrammarIntegration.java @@ -55,7 +55,7 @@ public class TestElasticsearchExecutionPlanFromGrammarIntegration { - private static final String TEST_IMAGE_TAG = "7.8.0"; + private static final String TEST_IMAGE_TAG = "7.17.7"; private static PureModel PURE_MODEL; @BeforeClass