Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added support for Match function for Semi-Structured data. #2361

Merged
merged 7 commits into from
Oct 18, 2023
+1,158 −4
Merged

Added support for Match function for Semi-Structured data. #2361

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f33d448
Added Support for Match Function
pragyasri-gs Oct 11, 2023
e7fcf9d
Typo Fix.
pragyasri-gs Oct 12, 2023
86e4e4f
Type fix.
pragyasri-gs Oct 12, 2023
4dace36
Typo fix.
pragyasri-gs Oct 12, 2023
37aa710
Typo fix.
pragyasri-gs Oct 12, 2023
d0aabc7
Cast Fix.
pragyasri-gs Oct 12, 2023
51b4c50
Merge branch 'master' into dev-match
pragyasri-gs Oct 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ...e-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,11 @@ function <<access.private>> meta::pure::router::routing::routeValueSpecification
|$state,
|let processedValues = processCollection($state, $i.values->evaluateAndDeactivate(), $executionContext, $vars, $inScopeVars, v:Any[1]|true, $extensions, $debug);
let last = $processedValues->last()->toOne();
let shouldClean = $processedValues.value->evaluateAndDeactivate()->forAll(p|$p->instanceOf(InstanceValue) && $p->cast(@InstanceValue).genericType.rawType != LambdaFunction);
let shouldClean = $processedValues->evaluateAndDeactivate().value->forAll(p|$p->instanceOf(InstanceValue) && $p->cast(@InstanceValue).genericType.rawType != LambdaFunction);

^$last(value = if ($shouldClean,
| ^$i(values = $processedValues.value->evaluateAndDeactivate()->cast(@InstanceValue)->map(e|$e.values)),
| ^$i(values = $processedValues.value)));
| ^$i(values = $processedValues->evaluateAndDeactivate().value->cast(@InstanceValue)->map(e|$e.values)),
| ^$i(values = $processedValues->evaluateAndDeactivate().value)));
);
);,
cs:ClassSetImplementationHolder[1] | let param = $cs.value->cast(@InstanceValue).values->at(0)->cast(@Class<Any>); // TODO: cleanup needed
Expand Down
177 changes: 177 additions & 0 deletions .../execution/stores/relational/test/semiStructured/TestSnowflakeSemiStructuredMatching.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
// Copyright 2022 Goldman Sachs
pragyasri-gs marked this conversation as resolved.
Show resolved Hide resolved
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.finos.legend.engine.plan.execution.stores.relational.test.semiStructured;

import org.eclipse.collections.impl.multimap.set.SynchronizedSetMultimap;
import org.junit.Assert;
import org.junit.Test;

public class TestSnowflakeSemiStructuredMatching extends AbstractTestSnowflakeSemiStructured
{
private static final String snowflakeMapping = "match::mapping::SnowflakeMapping";
private static final String snowflakeRuntime = "match::runtime::SnowflakeRuntime";

@Test
public void testSemiStructuredMatchComplexProperty()
{
String queryFunction = "match::semiStructuredMatchComplexProperty__TabularDataSet_1_";
String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime);
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Customer Address, String, \"\", \"\")]\n" +
" resultColumns = [(\"Customer Address\", \"\")]\n" +
" sql = select case when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('BillingAddress') then \"root\".CUSTOMER['customerAddress']['billAddress']::varchar when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('ShippingAddress') then \"root\".CUSTOMER['customerAddress']['shipAddress']::varchar else 'Default Address' end as \"Customer Address\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Customer Address, String, \"\", \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}


@Test
public void testSemiStructuredMatchWithMultipleProject()
{
String queryFunction = "match::semiStructuredMatchWithMultipleProject__TabularDataSet_1_";
String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime);
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n" +
" resultColumns = [(\"Customer Address\", \"\"), (\"Order Price\", \"\")]\n" +
" sql = select case when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('BillingAddress') then \"root\".CUSTOMER['customerAddress']['billAddress']::varchar when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('ShippingAddress') then \"root\".CUSTOMER['customerAddress']['shipAddress']::varchar else null end as \"Customer Address\", case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] else null end as \"Order Price\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}

@Test
public void testSemiStructuredMatchWithComplexFilter()
{
String queryFunction = "match::semiStructuredMatchWithComplexFilter__TabularDataSet_1_";
String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime);
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Customer Address, String, \"\", \"\")]\n" +
" resultColumns = [(\"Customer Address\", \"\")]\n" +
" sql = select case when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('BillingAddress') then \"root\".CUSTOMER['customerAddress']['billAddress']::varchar when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('ShippingAddress') then \"root\".CUSTOMER['customerAddress']['shipAddress']::varchar else null end as \"Customer Address\" from ORDER_SCHEMA.ORDER_TABLE as \"root\" where case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] else null end < 200\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Customer Address, String, \"\", \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}


@Test
public void testSemiStructuredMatchWithVariableAccess()
{
String queryFunction = "match::semiStructuredMatchWithVariableAccess__TabularDataSet_1_";
String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime);
String snowflakeExpected = "Sequence\n" +
"(\n" +
" type = TDS[(Max Amount Flag, Boolean, \"\", \"\")]\n" +
" (\n" +
" Allocation\n" +
" (\n" +
" type = Integer\n" +
" resultSizeRange = 1\n" +
" name = maxAmount\n" +
" value = \n" +
" (\n" +
" Constant\n" +
" (\n" +
" type = Integer\n" +
" resultSizeRange = 1\n" +
" values=[200]\n" +
" )\n" +
" )\n" +
" )\n" +
" RelationalBlockExecutionNode\n" +
" (\n" +
" type = TDS[(Max Amount Flag, Boolean, \"\", \"\")]\n" +
" (\n" +
" SQL\n" +
" (\n" +
" type = Void\n" +
" resultColumns = []\n" +
" sql = ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"${execID}\", \"engineUser\" : \"${userId}\", \"referer\" : \"${referer}\"}';\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n" +
" Relational\n" +
" (\n" +
" type = TDS[(Max Amount Flag, Boolean, \"\", \"\")]\n" +
" resultColumns = [(\"Max Amount Flag\", \"\")]\n" +
" sql = select case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then case when \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] < ${maxAmount} then 'true' else 'false' end when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] < ${maxAmount} then 'true' else 'false' end else null end as \"Max Amount Flag\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n" +
" ) \n" +
" finallyExecutionNodes = \n" +
" (\n" +
" SQL\n" +
" (\n" +
" type = Void\n" +
" resultColumns = []\n" +
" sql = ALTER SESSION UNSET QUERY_TAG;\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n" +
" )\n" +
" )\n" +
" )\n" +
")\n";
Assert.assertEquals(snowflakeExpected, snowflakePlan);
}


@Test
public void testSemiStructuredMatchMultilevel()
{
String queryFunction = "match::semiStructuredMatchMultilevel__TabularDataSet_1_";
String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime);
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Amount, Integer, \"\", \"\")]\n" +
" resultColumns = [(\"Amount\", \"\")]\n" +
" sql = select case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('WalletPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['walletTransactionAmount'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['cardTransactionAmount'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] else null end when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] else null end as \"Amount\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Amount, Integer, \"\", \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}

@Test
public void testSemiStructuredMatchWithMultipleProjectUsingCol()
{
String queryFunction = "match::semiStructuredMatchWithMultipleProjectUsingCol__TabularDataSet_1_";
String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime);
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n" +
" resultColumns = [(\"Customer Address\", \"\"), (\"Order Price\", \"\")]\n" +
" sql = select case when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('BillingAddress') then \"root\".CUSTOMER['customerAddress']['billAddress']::varchar when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('ShippingAddress') then \"root\".CUSTOMER['customerAddress']['shipAddress']::varchar else null end as \"Customer Address\", case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] else null end as \"Order Price\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}

public String modelResourcePath()
{
return "/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure";
}
}
Loading
Loading