From 5b6d7664b25b9e74336938f4de2b9ee332951baa Mon Sep 17 00:00:00 2001 From: Gopichand Kotana <109651657+gs-kotang@users.noreply.github.com> Date: Sat, 4 Nov 2023 15:51:10 +0530 Subject: [PATCH] Fix - In processing with Literal list For Snowflake (#2415) * Fix - In processing with Literal list * Add unit test - Execution plan test for In processing with literal list in snowflake * Refactor - Use a function that processes temp table name literal based on db type --- .../tests/executionPlanTestSnowflake.pure | 18 ++++++++++++++++++ .../processInOperation.pure | 13 +++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/executionPlan/tests/executionPlanTestSnowflake.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/executionPlan/tests/executionPlanTestSnowflake.pure index f0487a9aef8..d6f3b67a498 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/executionPlan/tests/executionPlanTestSnowflake.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/executionPlan/tests/executionPlanTestSnowflake.pure @@ -131,3 +131,21 @@ function <> meta::pure::executionPlan::tests::snowflake::testRelation let expectedPlan = 'Relational(type=TDS[(Name,String,VARCHAR(200),"")]resultColumns=[("Name",VARCHAR(200))]sql=select"root".NAMEas"Name"fromproductSchema.productTableas"root"connection=RelationalDatabaseConnection(type="Snowflake"))'; assertEquals($expectedPlan, $generatedPlan->planToStringWithoutFormatting(meta::relational::extension::relationalExtensions())); } + +function <> meta::pure::executionPlan::tests::snowflake::testInExecutionWithLiteralListSnowflake():Boolean[1] +{ + let intList = range(1,17000); + let generatedPlan = executionPlan({|Person.all()->filter(p|$p.age->in($intList))->project([x |$x.name], ['fullName'])}, simpleRelationalMapping, ^Runtime(connectionStores=^ConnectionStore(element = meta::relational::tests::db,connection=meta::pure::executionPlan::tests::snowflake::relationalConnectionForSnowflake(true))), meta::relational::extension::relationalExtensions()); + let execNodesInRelBlockNode = $generatedPlan.rootExecutionNode.executionNodes; + + let createAndPopulateTempTableNode = $execNodesInRelBlockNode->filter(e|$e->instanceOf(CreateAndPopulateTempTableExecutionNode)); + + let tempTableName = $createAndPopulateTempTableNode->at(0)->cast(@CreateAndPopulateTempTableExecutionNode).tempTableName; + assertEquals('LEGEND_TEMP_DB.LEGEND_TEMP_SCHEMA.tempTableForIn_8', $tempTableName); + let inputVarNames = $createAndPopulateTempTableNode->at(0)->cast(@CreateAndPopulateTempTableExecutionNode).inputVarNames; + assertEquals('tempVarForIn_8', $inputVarNames->at(0)); + + let allocationNode = $execNodesInRelBlockNode->filter(e|$e->instanceOf(AllocationExecutionNode)); + let varName = $allocationNode->at(0)->cast(@AllocationExecutionNode).varName; + assertEquals('tempVarForIn_8', $varName); +} \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/postprocessor/defaultPostProcessor/processInOperation.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/postprocessor/defaultPostProcessor/processInOperation.pure index 48938985b61..7f01ed701e0 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/postprocessor/defaultPostProcessor/processInOperation.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/postprocessor/defaultPostProcessor/processInOperation.pure @@ -52,6 +52,11 @@ function <> meta::relational::postProcessor::prefixForWrapperAll 'inFilterClause_'; } +function <> meta::relational::postProcessor::processedTempTableNameForIn(dbType: DatabaseType[1]):String[1] +{ + $dbType->createDbConfig([]).procesTempTableName('tempTableForIn_'); +} + function meta::relational::postProcessor::processInOperation(query:SQLQuery[1], runtime:Runtime[1], store:Database[0..1], exeCtx:ExecutionContext[1], extensions:Extension[*]):PostProcessorResult[1] { let connection = $runtime->connectionByElement($store->toOne())->meta::relational::mapping::updateConnection($extensions)->cast(@DatabaseConnection); @@ -68,12 +73,12 @@ function meta::relational::postProcessor::processInOperation(query:SQLQuery[1], d : DynaFunction[1] | if($d.name == 'in' && $d.parameters->at(1)->instanceOf(LiteralList) && ($d.parameters->at(1)->cast(@LiteralList).values->size() > $dbThreshold->toOne()), | let dbType = $connection.type; - let tempTableName = 'tempTableForIn_' + $uniqueId->toString(); + let tempTableName = $dbType->processedTempTableNameForIn() + $uniqueId->toString(); let tempTableColumnName = 'ColumnForStoringInCollection'; let firstLiteralValue = $d.parameters->at(1)->cast(@LiteralList).values->map(l | $l.value)->at(0); let collectionValueType = if($firstLiteralValue->instanceOf(VarPlaceHolder), | $firstLiteralValue->cast(@VarPlaceHolder).type, | $firstLiteralValue->type()); - let selectSQLQuery = generateTempTableSelectSQLQuery('default', $dbType->createDbConfig([]).procesTempTableName($tempTableName), $tempTableColumnName, meta::relational::transform::fromPure::pureTypeToDataTypeMap()->get($collectionValueType)->translateCoreTypeToDbSpecificType(^TranslationContext(dbType=$dbType))->toOne()); + let selectSQLQuery = generateTempTableSelectSQLQuery('default', $tempTableName, $tempTableColumnName, meta::relational::transform::fromPure::pureTypeToDataTypeMap()->get($collectionValueType)->translateCoreTypeToDbSpecificType(^TranslationContext(dbType=$dbType))->toOne()); ^$d(parameters = [$d.parameters->at(0), $selectSQLQuery]);, |$d);, @@ -112,8 +117,8 @@ function meta::relational::postProcessor::generatePostProcessorResult(changedFun let outerAllocationNodeName = if($newInFunction.parameters->at(1)->instanceOf(VarPlaceHolder), |$newInFunction.parameters->at(1)->cast(@VarPlaceHolder).name, - |$newInFunction.parameters->at(1)->cast(@SelectSQLQuery).data.alias.name->toOne()->replace('tempTableForIn_', prefixForWrapperAllocationNodeName())); - let tempTableName = $dbType->createDbConfig([]).procesTempTableName($outerAllocationNodeName->replace(prefixForWrapperAllocationNodeName(), 'tempTableForIn_')); + |$newInFunction.parameters->at(1)->cast(@SelectSQLQuery).data.alias.name->toOne()->replace($dbType->processedTempTableNameForIn(), prefixForWrapperAllocationNodeName())); + let tempTableName = $outerAllocationNodeName->replace(prefixForWrapperAllocationNodeName(), $dbType->processedTempTableNameForIn()); let tempTableColumnName = 'ColumnForStoringInCollection'; let allocationNodeName = $outerAllocationNodeName->replace(prefixForWrapperAllocationNodeName(), 'tempVarForIn_');