From d447ca4ae3676f83aa1c654ed57dc83d54ae8cbd Mon Sep 17 00:00:00 2001 From: "Girdhar, Tanuj [AM Public]" Date: Wed, 22 Nov 2023 00:31:20 -0500 Subject: [PATCH] add udf for flattening in H2 --- ...TestSnowflakeSemiStructuredFlattening.java | 16 + .../stores/relational/LegendH2Extensions.java | 166 ++++++- .../DefaultH2AuthenticationStrategy.java | 2 + .../TestExplodeSemiStructured.java | 246 ++++++++++ .../TestSemiStructuredFlattening.java | 245 ++++++++++ .../explodeSemiStructuredMapping.pure | 458 ++++++++++++++++++ .../semiStructuredFlattening.pure | 298 ++++++++++++ .../LegendH2Extensions_1_4_200.java | 163 +++++++ .../pureToSQLQuery/pureToSQLQuery.pure | 18 + .../dbSpecific/h2/h2Extension1_4_200.pure | 83 +++- .../dbSpecific/h2/h2Extension2_1_214.pure | 102 +++- 11 files changed, 1781 insertions(+), 16 deletions(-) create mode 100644 legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestExplodeSemiStructured.java create mode 100644 legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredFlattening.java create mode 100644 legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/explodeSemiStructuredMapping.pure create mode 100644 legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredFlattening.pure diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSnowflakeSemiStructuredFlattening.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSnowflakeSemiStructuredFlattening.java index 8a6a9d8b514..dbbf7b2594d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSnowflakeSemiStructuredFlattening.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSnowflakeSemiStructuredFlattening.java @@ -198,6 +198,22 @@ public void testSemiStructuredSubAggregationDeep() Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan); } + @Test + public void testSemiStructuredMultiLevelFlatten() + { + String snowflakePlan = this.buildExecutionPlanString("flatten::semiStructuredMultiLevelFlattening__TabularDataSet_1_", snowflakeMapping, snowflakeRuntime); + String snowflakeExpected = + " Relational\n" + + " (\n" + + " type = TDS[(First Name, String, VARCHAR(100), \"\"), (Firm Name, String, \"\", \"\"), (Firm Address Name Line No, Integer, \"\", \"\")]\n" + + " resultColumns = [(\"First Name\", VARCHAR(100)), (\"Firm Name\", \"\"), (\"Firm Address Name Line No\", \"\")]\n" + + " sql = select \"root\".FIRSTNAME as \"First Name\", \"root\".FIRM_DETAILS['legalName']::varchar as \"Firm Name\", \"ss_flatten_1\".value['lineno'] as \"Firm Address Name Line No\" from PERSON_SCHEMA.PERSON_TABLE as \"root\" inner join lateral flatten(input => \"root\".FIRM_DETAILS['addresses'], outer => true, recursive => false, mode => 'array') as \"ss_flatten_0\" inner join lateral flatten(input => \"ss_flatten_0\".value['lines'], outer => true, recursive => false, mode => 'array') as \"ss_flatten_1\"\n" + + " connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" + + " )\n"; + String TDSType = " type = TDS[(First Name, String, VARCHAR(100), \"\"), (Firm Name, String, \"\", \"\"), (Firm Address Name Line No, Integer, \"\", \"\")]\n"; + Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan); + } + @Test public void testSemiStructuredMultiFlatten() { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions.java index ec98e7b74b5..c0d03b3ea44 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions.java @@ -19,7 +19,9 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.StringUtils; import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.h2.tools.SimpleResultSet; import org.h2.value.Value; +import org.h2.value.ValueArray; import org.h2.value.ValueBigint; import org.h2.value.ValueBoolean; import org.h2.value.ValueDouble; @@ -28,9 +30,15 @@ import org.h2.value.ValueReal; import org.h2.value.ValueVarchar; +import java.math.BigDecimal; import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Types; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -132,12 +140,12 @@ public static String legend_h2_extension_reverse_string(String string) public static String legend_h2_extension_split_part(String string, String token, Integer part) { - if (part < 1) + if (part < 1) { throw new IllegalArgumentException("Split part must be greater than zero"); } - if (string == null) + if (string == null) { return null; } @@ -147,4 +155,158 @@ public static String legend_h2_extension_split_part(String string, String token, return parts.length > readjustedPart ? parts[readjustedPart] : null; } + + private static HashSet extractProperty(HashSet resultSet, Object pathToExtract) + { + HashSet res = new HashSet<>(); + if (pathToExtract instanceof String) + { + String property = (String) pathToExtract; + if (property.equals("*")) + { + for (Object r: resultSet) + { + if (!(r instanceof Iterable)) + { + continue; + } + for (Object o : (Iterable) r) + { + res.add(o); + } + } + } + else + { + for (Object r: resultSet) + { + try + { + Object o = ((HashMap)(r)).get(property); + if (o != null) + { + res.add(o); + } + } + catch (Exception e) + { + e.printStackTrace(); // don't stop execution + } + } + } + } + else + { + int index = (int) pathToExtract; + for (Object r: resultSet) + { + try + { + res.add(((ArrayList)(r)).get(index)); + } + catch (Exception e) + { + e.printStackTrace(); + } + + } + } + return res; + } + + public static ResultSet legend_h2_extension_flatten_array(Connection conn, String tableName, String toFlattenColumnName, ValueArray jsonPaths) + { + try + { + String sql = String.format("select distinct %s as flat from %s", toFlattenColumnName, tableName); + ResultSet resultSet = conn.createStatement().executeQuery(sql); + HashSet toFlatten = new HashSet<>(); + while (resultSet.next()) + { + String json = resultSet.getString("flat"); + toFlatten.add(OBJECT_MAPPER.readValue(json, HashMap.class)); + } + + ArrayList pathsToExtract = OBJECT_MAPPER.readValue(jsonPaths.getString(), ArrayList.class); + + for (Object path: pathsToExtract) + { + toFlatten = extractProperty(toFlatten, path); + } + + SimpleResultSet flattenedResultSet = new SimpleResultSet(); + + flattenedResultSet.addColumn("__INPUT__", Types.VARCHAR, 1000, 0); // using the original array as joinKey + + // use first non-null object to infer the type of value + boolean resolvedFlattenedType = false; + + for (Object o: toFlatten) + { + if (!(o instanceof Iterable)) + { + continue; + } + for (Object value : (Iterable) o) + { + if (value instanceof Map || value instanceof List || value instanceof String) + { + flattenedResultSet.addColumn("VALUE", Types.VARCHAR, 1000, 0); + } + else if (value instanceof Boolean) + { + flattenedResultSet.addColumn("VALUE", Types.BOOLEAN, 0, 0); + } + else if (value instanceof Double || value instanceof Float || value instanceof BigDecimal) + { + flattenedResultSet.addColumn("VALUE", Types.DOUBLE, 20, 20); + } + else if (value instanceof Integer || value instanceof Long) + { + flattenedResultSet.addColumn("VALUE", Types.BIGINT, 0, 0); + } + else + { + throw new RuntimeException("unsupported data type in h2 extension"); + } + resolvedFlattenedType = true; + break; + } + if (resolvedFlattenedType) + { + break; + } + } + + if (!resolvedFlattenedType) + { + flattenedResultSet.addColumn("VALUE", Types.VARCHAR, 1000, 0); + return flattenedResultSet; + } + + for (Object o: toFlatten) + { + if (!(o instanceof Iterable)) + { + continue; + } + for (Object value : (Iterable) o) + { + if (value instanceof Map || value instanceof List) + { + flattenedResultSet.addRow(Arrays.asList(OBJECT_MAPPER.writeValueAsString(o), OBJECT_MAPPER.writeValueAsString(value)).toArray()); + } + else + { + flattenedResultSet.addRow(Arrays.asList(OBJECT_MAPPER.writeValueAsString(o), value).toArray()); + } + } + } + return flattenedResultSet; + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/authentication/strategy/DefaultH2AuthenticationStrategy.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/authentication/strategy/DefaultH2AuthenticationStrategy.java index f63987ef7bd..8e919a9f625 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/authentication/strategy/DefaultH2AuthenticationStrategy.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/authentication/strategy/DefaultH2AuthenticationStrategy.java @@ -107,6 +107,7 @@ private static List getLegendH2ExtensionSQLs() "CREATE ALIAS IF NOT EXISTS legend_h2_extension_base64_decode FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_base64_decode\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_base64_encode FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_base64_encode\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_reverse_string FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_reverse_string\";", + "CREATE ALIAS IF NOT EXISTS legend_h2_extension_flatten_array FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_flatten_array\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_split_part FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_split_part\";" ); } @@ -121,6 +122,7 @@ private static List getLegendH2_1_4_200_ExtensionSQLs() "CREATE ALIAS IF NOT EXISTS legend_h2_extension_reverse_string FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_reverse_string\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_hash_md5 FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_hash_md5\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_hash_sha1 FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_hash_sha1\";", + "CREATE ALIAS IF NOT EXISTS legend_h2_extension_flatten_array FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_flatten_array\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_split_part FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_split_part\";" ); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestExplodeSemiStructured.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestExplodeSemiStructured.java new file mode 100644 index 00000000000..81835786adc --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestExplodeSemiStructured.java @@ -0,0 +1,246 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.plan.execution.stores.relational.test.semiStructured; + +import org.junit.Assert; +import org.junit.Test; + +public class TestExplodeSemiStructured extends AbstractTestSemiStructured +{ + private static final String h2Mapping = "simple::mapping::semistructured"; + private static final String h2Runtime = "simple::runtime::runtime"; + + @Test + public void testSimplePrimitivePropertiesProjectExplodeSource() + { + String queryFunction = "simple::query::getOrdersForBlock__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("1,a1,o1,i1,100.0\n" + + "2,a1,o2,i2,10.0\n" + + "2,a1,o1,i1,100.0\n" + + "3,a2,o3,i3,100.0\n" + + "4,a2,o4,i1,100.0\n" + + "5,a1,o5,i4,50.0\n" + + "5,a1,o6,i4,50.0\n" + + "6,a3,,,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSimplePrimitivePropertiesProjectExplodeTarget() + { + String queryFunction = "simple::query::getBlockForTrade__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("t1,accepted,1,a1\n" + + "t2,rejected,2,a1\n" + + "t3,accepted,3,a2\n" + + "t4,accepted,3,a2\n" + + "t5,accepted,4,a2\n" + + "t6,rejected,4,a2\n" + + "t7,accepted,5,a1\n" + + "t8,invalid,,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testComplexProjectFlattenedAndExplodedPropertiesInProject() + { + String queryFunction = "simple::query::getOrdersAndRelatedEntitiesForBlock__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("1,trade,t1,o1\n" + + "1,order,o1,o1\n" + + "2,trade,t2,o2\n" + + "2,trade,t2,o1\n" + + "2,order,o2,o2\n" + + "2,order,o2,o1\n" + + "2,order,o1,o2\n" + + "2,order,o1,o1\n" + + "3,trade,t3,o3\n" + + "3,trade,t4,o3\n" + + "3,order,o3,o3\n" + + "4,trade,t5,o4\n" + + "4,trade,t6,o4\n" + + "4,order,o4,o4\n" + + "5,trade,t7,o5\n" + + "5,trade,t7,o6\n" + + "5,order,o5,o5\n" + + "5,order,o5,o6\n" + + "5,order,o6,o5\n" + + "5,order,o6,o6\n" + + "6,,,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testComplexProjectMultiplePropertiesToExplodeInProject() + { + String queryFunction = "simple::query::getTradesAndOrdersInBlock__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("1,a1,o1,i1,t1,accepted\n" + + "2,a1,o2,i2,t2,rejected\n" + + "2,a1,o1,i1,t2,rejected\n" + + "3,a2,o3,i3,t3,accepted\n" + + "3,a2,o3,i3,t4,accepted\n" + + "4,a2,o4,i1,t5,accepted\n" + + "4,a2,o4,i1,t6,rejected\n" + + "5,a1,o5,i4,t7,accepted\n" + + "5,a1,o6,i4,t7,accepted\n" + + "6,a3,,,,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSimplePrimitivePropertiesProjectWithFilterOnSource() + { + String queryFunction = "simple::query::getTradesForNonCancelledBlocks__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("1,a1,t1,accepted\n" + + "3,a2,t3,accepted\n" + + "3,a2,t4,accepted\n" + + "4,a2,t5,accepted\n" + + "4,a2,t6,rejected\n" + + "5,a1,t7,accepted\n" + + "6,a3,,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSimplePrimitivePropertiesProjectWithFilterOnTarget() + { + String queryFunction = "simple::query::getNonCancelledBlocksForTrades__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("t1,accepted,1,a1\n" + + "t3,accepted,3,a2\n" + + "t4,accepted,3,a2\n" + + "t5,accepted,4,a2\n" + + "t6,rejected,4,a2\n" + + "t7,accepted,5,a1\n" + + "t8,invalid,,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testProjectWithExplodedPropertyAccessOnlyInFilter() + { + String queryFunction = "simple::query::getNonCancelledBlocksForTradesNoProject__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("t1,accepted\n" + + "t3,accepted\n" + + "t4,accepted\n" + + "t5,accepted\n" + + "t6,rejected\n" + + "t7,accepted\n" + + "t8,invalid\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testFilterOnExplodedPropertyFilteringInsideProject() + { + String queryFunction = "simple::query::getBigBuyOrdersInBlock__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("1,a1,,o1\n" + + "2,a1,o2,o2\n" + + "2,a1,o2,o1\n" + + "3,a2,o3,o3\n" + + "4,a2,,o4\n" + + "5,a1,,o5\n" + + "5,a1,,o6\n" + + "6,a3,,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testAggregationAggregateExplodedPropertyUsingGroupBy() + { + String queryFunction = "simple::query::getTradeVolumeInBlock__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("1,a1,100\n" + + "2,a1,100\n" + + "3,a2,200\n" + + "4,a2,150\n" + + "5,a1,60\n" + + "6,a3,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testAggregationAggregateExplodedPropertyInsideProject() + { + String queryFunction = "simple::query::getTotalBuyOrderVolumeInBlock__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("1,a1,\n" + + "2,a1,100\n" + + "3,a2,200\n" + + "4,a2,\n" + + "5,a1,\n" + + "6,a3,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSimpleJoinChainOneJoin() + { + String queryFunction = "simple::query::getAccountForOrders__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("o1,a1,1\n" + + "o1,a1,2\n" + + "o2,a1,2\n" + + "o3,a2,3\n" + + "o4,a2,4\n" + + "o5,a1,5\n" + + "o6,a1,5\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testJoinChainMultipleJoinsSingleExplode() + { + String queryFunction = "simple::query::getProductsForOrdersInBlock__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("p1,1\n" + + "p2,2\n" + + "p1,2\n" + + "p3,3\n" + + "p1,4\n" + + "p1,5\n" + + "p1,5\n" + + ",6\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testJoinChainMultipleJoinsMultipleExplode() + { + String queryFunction = "simple::query::getRelatedTradesForOrder__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("o1,t1\n" + + "o1,t2\n" + + "o2,t2\n" + + "o3,t3\n" + + "o3,t4\n" + + "o4,t5\n" + + "o4,t6\n" + + "o5,t7\n" + + "o6,t7\n", h2Result.replace("\r\n", "\n")); + } + + @Override + public String modelResourcePath() + { + return "/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/explodeSemiStructuredMapping.pure"; + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredFlattening.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredFlattening.java new file mode 100644 index 00000000000..3bd0f8f97bc --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredFlattening.java @@ -0,0 +1,245 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.plan.execution.stores.relational.test.semiStructured; + +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +public class TestSemiStructuredFlattening extends AbstractTestSemiStructured +{ + private static final String h2Mapping = "flatten::mapping::H2Mapping"; + private static final String h2Runtime = "flatten::runtime::H2Runtime"; + + @Test + public void testSemiStructuredPrimitivePropertyFlattening() + { + String queryFunction = "flatten::semiStructuredPrimitivePropertyFlattening__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,Firm X,O1\n" + + "Peter,Firm X,O2\n" + + "John,Firm X,O1\n" + + "John,Firm X,O2\n" + + "John,Firm X,O1\n" + + "John,Firm X,O2\n" + + "Anthony,Firm X,O1\n" + + "Anthony,Firm X,O2\n" + + "Fabrice,Firm A,O3\n" + + "Fabrice,Firm A,O4\n" + + "Oliver,Firm B,O5\n" + + "Oliver,Firm B,O6\n" + + "David,Firm B,O5\n" + + "David,Firm B,O6\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredComplexPropertyFlattening() + { + String queryFunction = "flatten::semiStructuredComplexPropertyFlattening__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,Firm X,A1\n" + + "Peter,Firm X,A11\n" + + "John,Firm X,A2\n" + + "John,Firm X,A22\n" + + "John,Firm X,A3\n" + + "John,Firm X,A32\n" + + "Anthony,Firm X,A4\n" + + "Fabrice,Firm A,A5\n" + + "Fabrice,Firm A,A52\n" + + "Oliver,Firm B,A6\n" + + "David,Firm B,A7\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredPrimitivePropertyArrayIndexing() + { + String queryFunction = "flatten::semiStructuredPrimitivePropertyArrayIndexing__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,Firm X,O1,\n" + + "John,Firm X,O1,\n" + + "John,Firm X,O1,\n" + + "Anthony,Firm X,O1,\n" + + "Fabrice,Firm A,O3,\n" + + "Oliver,Firm B,O5,\n" + + "David,Firm B,O5,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredComplexPropertyArrayIndexing() + { + String queryFunction = "flatten::semiStructuredComplexPropertyArrayIndexing__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,Firm X,A1,\n" + + "John,Firm X,A2,\n" + + "John,Firm X,A3,\n" + + "Anthony,Firm X,A4,\n" + + "Fabrice,Firm A,A5,\n" + + "Oliver,Firm B,A6,\n" + + "David,Firm B,A7,\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredComplexPropertyFlatteningFollowedBySubType() + { + String queryFunction = "flatten::semiStructuredComplexPropertyFlatteningFollowedBySubType__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,Firm X,1\n" + + "Peter,Firm X,1\n" + + "John,Firm X,1\n" + + "John,Firm X,1\n" + + "John,Firm X,1\n" + + "John,Firm X,1\n" + + "Anthony,Firm X,1\n" + + "Fabrice,Firm A,1\n" + + "Fabrice,Firm A,1\n" + + "Oliver,Firm B,1\n" + + "David,Firm B,1\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredComplexPropertyArrayIndexingFollowedBySubType() + { + String queryFunction = "flatten::semiStructuredComplexPropertyArrayIndexingFollowedBySubType__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,Firm X,1\n" + + "John,Firm X,1\n" + + "John,Firm X,1\n" + + "Anthony,Firm X,1\n" + + "Fabrice,Firm A,1\n" + + "Oliver,Firm B,1\n" + + "David,Firm B,1\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredPrimitivePropertyFiltering() + { + String queryFunction = "flatten::semiStructuredPrimitivePropertyFiltering__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("", h2Result.replace("\r\n", "\n")); + } + + @Test + @Ignore + public void testSemiStructuredPrimitivePropertyFilteringInProject() + { + String queryFunction = "flatten::semiStructuredPrimitivePropertyFilteringInProject__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredSubAggregation() + { + String queryFunction = "flatten::semiStructuredSubAggregation__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,Firm X,A1;A11\n" + + "John,Firm X,A2;A22\n" + + "John,Firm X,A3;A32\n" + + "Anthony,Firm X,A4\n" + + "Fabrice,Firm A,A5;A52\n" + + "Oliver,Firm B,A6\n" + + "David,Firm B,A7\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredSubAggregationDeep() + { + String queryFunction = "flatten::semiStructuredSubAggregationDeep__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,Firm X,6\n" + + "John,Firm X,6\n" + + "John,Firm X,6\n" + + "Anthony,Firm X,3\n" + + "Fabrice,Firm A,6\n" + + "Oliver,Firm B,3\n" + + "David,Firm B,3\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredMultiLevelFlatten() + { + String queryFunction = "flatten::semiStructuredMultiLevelFlattening__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,Firm X,1\n" + + "Peter,Firm X,2\n" + + "Peter,Firm X,1\n" + + "Peter,Firm X,2\n" + + "John,Firm X,1\n" + + "John,Firm X,2\n" + + "John,Firm X,1\n" + + "John,Firm X,2\n" + + "John,Firm X,1\n" + + "John,Firm X,2\n" + + "John,Firm X,1\n" + + "John,Firm X,2\n" + + "Anthony,Firm X,1\n" + + "Anthony,Firm X,2\n" + + "Fabrice,Firm A,1\n" + + "Fabrice,Firm A,2\n" + + "Fabrice,Firm A,1\n" + + "Fabrice,Firm A,2\n" + + "Oliver,Firm B,1\n" + + "Oliver,Firm B,2\n" + + "David,Firm B,1\n" + + "David,Firm B,2\n", h2Result.replace("\r\n", "\n")); + } + + @Test + public void testSemiStructuredMultiFlatten() + { + String queryFunction = "flatten::semiStructuredMultiFlatten__TabularDataSet_1_"; + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("Peter,A1,1,O1\n" + + "Peter,A1,1,O2\n" + + "Peter,A11,1,O1\n" + + "Peter,A11,1,O2\n" + + "John,A2,1,O1\n" + + "John,A2,1,O2\n" + + "John,A22,1,O1\n" + + "John,A22,1,O2\n" + + "John,A3,1,O1\n" + + "John,A3,1,O2\n" + + "John,A32,1,O1\n" + + "John,A32,1,O2\n" + + "Anthony,A4,1,O1\n" + + "Anthony,A4,1,O2\n" + + "Fabrice,A5,1,O3\n" + + "Fabrice,A5,1,O4\n" + + "Fabrice,A52,1,O3\n" + + "Fabrice,A52,1,O4\n" + + "Oliver,A6,1,O5\n" + + "Oliver,A6,1,O6\n" + + "David,A7,1,O5\n" + + "David,A7,1,O6\n", h2Result.replace("\r\n", "\n")); + } + + @Override + public String modelResourcePath() + { + return "/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredFlattening.pure"; + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/explodeSemiStructuredMapping.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/explodeSemiStructuredMapping.pure new file mode 100644 index 00000000000..466845de85c --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/explodeSemiStructuredMapping.pure @@ -0,0 +1,458 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +###Pure +Class simple::model::order +{ + id: String[1]; + identifier: String[1]; + quantity: Integer[1]; + side: String[1]; + price: Float[1]; + account: String[1]; + tradeId: String[1]; + product: simple::model::Product[1]; + block: simple::model::block[1]; +} + +Class simple::model::trade +{ + id: String[1]; + tradeSummary: simple::model::tradeSummary[1]; + status: String[1]; + block: simple::model::block[1]; +} + +Class simple::model::tradeSummary +{ + identifier: String[1]; + execQuantity: Integer[1]; + price: String[1]; + side: String[1]; + orderId: String[1]; +} + +Class simple::model::block +{ + id: String[1]; + account: String[1]; + blockData: simple::model::blockData[1]; + trades: simple::model::trade[*]; + orders: simple::model::order[*]; + products: String[*]; +} + +Class simple::model::relatedEntity +{ + tag: String[1]; + tagId: String[1]; +} + +Class simple::model::blockData +{ + relatedEntities: simple::model::relatedEntity[1..*]; + status: String[1]; +} + +Class simple::model::Product +{ + product: String[1]; + identifier: String[1]; + version: String[1]; +} + +###ExternalFormat +Binding simple::binding::binding +{ + contentType: 'application/json'; + modelIncludes: [ + simple::model::tradeSummary, + simple::model::relatedEntity, + simple::model::blockData + ]; +} + +###Relational +Database simple::store::semistructured +( + Schema Semistructured + ( + Table Blocks + ( + ID varchar(100) PRIMARY KEY, + ACCOUNT varchar(100), + BLOCKDATA SEMISTRUCTURED + ) + Table Trades + ( + ID varchar(100) PRIMARY KEY, + STATUS varchar(100), + TRADESUMMARY SEMISTRUCTURED + ) + Table Orders + ( + ID varchar(100) PRIMARY KEY, + IDENTIFIER varchar(100), + QUANTITY INTEGER, + SIDE varchar(100), + PRICE DOUBLE + ) + Table Product + ( + IDENTIFIER varchar(100) PRIMARY KEY, + VERSION varchar(100) PRIMARY KEY, + PRODUCT varchar(100) + ) + ) + + Join Block_Order(extractFromSemiStructured(explodeSemiStructured(Semistructured.Blocks.BLOCKDATA, 'relatedEntities', 'SEMISTRUCTURED'), 'tag', 'VARCHAR') = 'order' + and extractFromSemiStructured(explodeSemiStructured(Semistructured.Blocks.BLOCKDATA, 'relatedEntities', 'SEMISTRUCTURED'), 'tagId', 'VARCHAR') = Semistructured.Orders.ID) + Join Block_Trade(extractFromSemiStructured(explodeSemiStructured(Semistructured.Blocks.BLOCKDATA, 'relatedEntities', 'SEMISTRUCTURED'), 'tag', 'VARCHAR') = 'trade' + and extractFromSemiStructured(explodeSemiStructured(Semistructured.Blocks.BLOCKDATA, 'relatedEntities', 'SEMISTRUCTURED'), 'tagId', 'VARCHAR') = Semistructured.Trades.ID) + Join Order_Product(Semistructured.Orders.IDENTIFIER = Semistructured.Product.IDENTIFIER) +) + +###Mapping +Mapping simple::mapping::semistructured +( + *simple::model::block: Relational + { + ~primaryKey + ( + [simple::store::semistructured]Semistructured.Blocks.ID + ) + ~mainTable [simple::store::semistructured]Semistructured.Blocks + id: [simple::store::semistructured]Semistructured.Blocks.ID, + account: [simple::store::semistructured]Semistructured.Blocks.ACCOUNT, + trades[simple_model_trade]: [simple::store::semistructured]@Block_Trade, + orders[simple_model_order]: [simple::store::semistructured]@Block_Order, + products: [simple::store::semistructured]@Block_Order > [simple::store::semistructured]@Order_Product | [simple::store::semistructured]Semistructured.Product.PRODUCT, + blockData: Binding simple::binding::binding : [simple::store::semistructured]Semistructured.Blocks.BLOCKDATA + } + *simple::model::order: Relational + { + ~primaryKey + ( + [simple::store::semistructured]Semistructured.Orders.ID + ) + ~mainTable [simple::store::semistructured]Semistructured.Orders + id: [simple::store::semistructured]Semistructured.Orders.ID, + quantity: [simple::store::semistructured]Semistructured.Orders.QUANTITY, + side: [simple::store::semistructured]Semistructured.Orders.SIDE, + price: [simple::store::semistructured]Semistructured.Orders.PRICE, + block[simple_model_block]: [simple::store::semistructured]@Block_Order, + account: [simple::store::semistructured]@Block_Order | [simple::store::semistructured]Semistructured.Blocks.ACCOUNT, + tradeId: [simple::store::semistructured]@Block_Order > [simple::store::semistructured]@Block_Trade | [simple::store::semistructured]Semistructured.Trades.ID, + identifier: [simple::store::semistructured]Semistructured.Orders.IDENTIFIER, + product: [simple::store::semistructured]@Order_Product + } + *simple::model::trade: Relational + { + ~primaryKey + ( + [simple::store::semistructured]Semistructured.Trades.ID + ) + ~mainTable [simple::store::semistructured]Semistructured.Trades + id: [simple::store::semistructured]Semistructured.Trades.ID, + status: [simple::store::semistructured]Semistructured.Trades.STATUS, + block[simple_model_block]: [simple::store::semistructured]@Block_Trade, + tradeSummary: Binding simple::binding::binding : [simple::store::semistructured]Semistructured.Trades.TRADESUMMARY + } + *simple::model::Product: Relational + { + ~primaryKey + ( + [simple::store::semistructured]Semistructured.Product.IDENTIFIER + ) + ~mainTable [simple::store::semistructured]Semistructured.Product + identifier: [simple::store::semistructured]Semistructured.Product.IDENTIFIER, + version: [simple::store::semistructured]Semistructured.Product.VERSION, + product: [simple::store::semistructured]Semistructured.Product.PRODUCT + } +) + +###Connection +RelationalDatabaseConnection simple::connection::snowflake +{ + store: simple::store::semistructured; + type: H2; + specification: LocalH2 + { + testDataSetupSqls: + [ + 'drop schema if exists semistructured cascade;', + 'create schema semistructured;', + 'create table if not exists semistructured.blocks ( id int primary key, account varchar(100), blockData varchar(1000) ) as values (1, \'a1\', \'{"status": "fulfilled", "relatedEntities": [{"tag": "trade", "tagId": "t1"}, {"tag": "order", "tagId": "o1"}]}\'), (2, \'a1\', \'{"status": "cancelled", "relatedEntities": [{"tag": "trade", "tagId": "t2"}, {"tag": "order", "tagId": "o2"}, {"tag": "order", "tagId": "o1"}]}\'), (3, \'a2\', \'{"status": "fulfilled", "relatedEntities": [{"tag": "trade", "tagId": "t3"}, {"tag": "trade", "tagId": "t4"}, {"tag": "order", "tagId": "o3"}]}\'), (4, \'a2\', \'{"status": "accepted", "relatedEntities": [{"tag": "trade", "tagId": "t5"}, {"tag": "trade", "tagId": "t6"}, {"tag": "order", "tagId": "o4"}]}\'), (5, \'a1\', \'{"status": "fulfilled", "relatedEntities": [{"tag": "trade", "tagId": "t7"}, {"tag": "order", "tagId": "o5"}, {"tag": "order", "tagId": "o6"}]}\'), (6, \'a3\', \'{"status": "rejected", "relatedEntities": []}\');', + 'create table if not exists semistructured.trades ( id varchar(100) primary key, status varchar(100), tradeSummary varchar(1000) ) as values (\'t1\', \'accepted\', \'{"identifier": "i1", "execQuantity": 100, "execPrice": 100, "side": "SELL"}\'), (\'t2\', \'rejected\', \'{"identifier": "i2", "execQuantity": 100, "execPrice": 10, "side": "BUY"}\'), (\'t3\', \'accepted\', \'{"identifier": "i3", "execQuantity": 100, "execPrice": 100, "side": "BUY"}\'), (\'t4\', \'accepted\', \'{"identifier": "i3", "execQuantity": 100, "execPrice": 105, "side": "BUY"}\'), (\'t5\', \'accepted\', \'{"identifier": "i1", "execQuantity": 70, "execPrice": 100, "side": "SELL"}\'), (\'t6\', \'rejected\', \'{"identifier": "i1", "execQuantity": 80, "execPrice": 90, "side": "SELL"}\'), (\'t7\', \'accepted\', \'{"identifier": "i4", "execQuantity": 60, "execPrice": 50, "side": "SELL"}\'), (\'t8\', \'invalid\', \'{"identifier": "i4", "execQuantity": 60, "execPrice": 50, "side": "SELL"}\');', + 'create table if not exists semistructured.orders ( id varchar(100) primary key, identifier varchar(100), quantity int, side varchar(10), price double ) as values (\'o1\', \'i1\', 100, \'SELL\', 100), (\'o2\', \'i2\', 100, \'BUY\', 10), (\'o3\', \'i3\', 200, \'BUY\', 100), (\'o4\', \'i1\', 150, \'SELL\', 100), (\'o5\', \'i4\', 60, \'SELL\', 50), (\'o6\', \'i4\', 60, \'SELL\', 50);', + 'create table if not exists semistructured.product ( product varchar(100), identifier varchar(100), version varchar(100)) as values (\'p1\', \'i1\', \'v1\'), (\'p2\', \'i2\', \'v1\'), (\'p3\', \'i3\', \'v1\'), (\'p1\', \'i4\', \'v1\');' + ]; + }; + auth: Test; +} + +###Runtime +Runtime simple::runtime::runtime +{ + mappings: + [ + simple::mapping::semistructured + ]; + connections: + [ + simple::store::semistructured: + [ + connection_1: simple::connection::snowflake + ] + ]; +} + +###Pure +function simple::query::getOrdersForBlock():TabularDataSet[1] +{ + simple::model::block.all()->project( + [ + x|$x.id, + x|$x.account, + x|$x.orders.id, + x|$x.orders.identifier, + x|$x.orders.price + ], + [ + 'Id', + 'Account', + 'Orders/Id', + 'Orders/Identifier', + 'Orders/Price' + ] + ); +} + +function simple::query::getBlockForTrade():TabularDataSet[1] +{ + simple::model::trade.all()->project( + [ + x|$x.id, + x|$x.status, + x|$x.block.id, + x|$x.block.account + ], + [ + 'Id', + 'Status', + 'Block/Id', + 'Block/Account' + ] + ); +} + +function simple::query::getTradesForNonCancelledBlocks():TabularDataSet[1] +{ + simple::model::block.all()->filter( + x|!($x.blockData.status == 'cancelled') + )->project( + [ + x|$x.id, + x|$x.account, + x|$x.trades.id, + x|$x.trades.status + ], + [ + 'Id', + 'Account', + 'Trades/Id', + 'Trades/Status' + ] + ); +} + +function simple::query::getNonCancelledBlocksForTrades():TabularDataSet[1] +{ + simple::model::trade.all()->filter( + x|!($x.block.blockData.status == 'cancelled') + )->project( + [ + x|$x.id, + x|$x.status, + x|$x.block.id, + x|$x.block.account + ], + [ + 'Id', + 'Status', + 'Block/Id', + 'Block/Account' + ] + ); +} + +function simple::query::getNonCancelledBlocksForTradesNoProject():TabularDataSet[1] +{ + simple::model::trade.all()->filter( + x|!($x.block.blockData.status == 'cancelled') + )->project( + [ + x|$x.id, + x|$x.status + ], + [ + 'Id', + 'Status' + ] + ); +} + +function simple::query::getOrdersAndRelatedEntitiesForBlock():TabularDataSet[1] +{ + simple::model::block.all()->project( + [ + x|$x.id, + x|$x.blockData.relatedEntities.tag, + x|$x.blockData.relatedEntities.tagId, + x|$x.orders.id + ], + [ + 'Id', + 'Entity Tag', + 'Entity Tag Id', + 'Orders/Id' + ] + ); +} + +function simple::query::getBigBuyOrdersInBlock():TabularDataSet[1] +{ + simple::model::block.all()->project( + [ + x|$x.id, + x|$x.account, + x|$x.orders->filter( + x_1|($x_1.quantity >= 100) && + ($x_1.side == 'BUY') + ).id, + x|$x.orders.id + ], + [ + 'Block/Id', + 'Block/Account', + 'Big Buy Orders', + 'Orders/Id' + ] + ); +} + +function simple::query::getTradeVolumeInBlock():TabularDataSet[1] +{ + simple::model::block.all()->groupBy( + [ + x|$x.id, + x|$x.account + ], + [ + agg( + x|$x.trades.tradeSummary.execQuantity, + x|$x->sum() + ) + ], + [ + 'Id', + 'Account', + 'quantity' + ] + ); +} + +function simple::query::getTotalBuyOrderVolumeInBlock():TabularDataSet[1] +{ + simple::model::block.all()->project( + [ + x|$x.id, + x|$x.account, + x|$x.orders->filter( + x_1|$x_1.side == 'BUY' + ).quantity->sum() + ], + [ + 'Id', + 'Account', + 'Buy Order' + ] + ); +} + +function simple::query::getTradesAndOrdersInBlock():TabularDataSet[1] +{ + simple::model::block.all()->project( + [ + x|$x.id, + x|$x.account, + x|$x.orders.id, + x|$x.orders.identifier, + x|$x.trades.id, + x|$x.trades.status + ], + [ + 'Id', + 'Account', + 'Orders/Id', + 'Orders/Identifier', + 'Trades/Id', + 'Trades/Status' + ] + ); +} + +function simple::query::getAccountForOrders():TabularDataSet[1] +{ + simple::model::order.all()->project( + [ + x|$x.id, + x|$x.account, + x|$x.block.id + ], + [ + 'Id', + 'Account', + 'Block/Id' + ] + ); +} + +function simple::query::getRelatedTradesForOrder():TabularDataSet[1] +{ + simple::model::order.all()->project( + [ + x|$x.id, + x|$x.tradeId + ], + [ + 'Id', + 'Trade Id' + ] + ); +} + +function simple::query::getProductsForOrdersInBlock():TabularDataSet[1] +{ + simple::model::block.all()->project( + [ + x|$x.products, + x|$x.id + ], + [ + 'Product', + 'Id' + ] + ); +} \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredFlattening.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredFlattening.pure new file mode 100644 index 00000000000..87dec296fd3 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredFlattening.pure @@ -0,0 +1,298 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +###Pure +Class flatten::model::Person +{ + firstName: String[1]; + lastName: String[1]; + firm: flatten::model::Firm[1]; + manager: flatten::model::Person[1]; +} + +Class flatten::model::Firm +{ + legalName: String[1]; + otherNames: String[*]; + addresses: flatten::model::Address[*]; +} + +Class flatten::model::Address +{ + name: String[1]; +} + +Class flatten::model::AddressWithLines extends flatten::model::Address +{ + lines: flatten::model::AddressLine[*]; +} + +Class flatten::model::AddressLine +{ + lineno: Integer[1]; +} + +Class flatten::model::StreetAddressLine extends flatten::model::AddressLine +{ + street: String[1]; +} + +Class flatten::model::CityAddressLine extends flatten::model::AddressLine +{ + city: String[1]; +} + +Class flatten::model::StateAddressLine extends flatten::model::AddressLine +{ + state: String[1]; +} + +###Relational +Database flatten::store::H2DB +( + Schema PERSON_SCHEMA + ( + Table PERSON_TABLE + ( + ID INTEGER PRIMARY KEY, + FIRSTNAME VARCHAR(100), + LASTNAME VARCHAR(100), + FIRM_DETAILS VARCHAR(1000), + MANAGERID INTEGER + ) + ) + + Join manager(PERSON_SCHEMA.PERSON_TABLE.MANAGERID = {target}.ID) +) + + +###ExternalFormat +Binding flatten::store::FirmBinding +{ + contentType: 'application/json'; + modelIncludes: [ + flatten::model::Firm, + flatten::model::Address, + flatten::model::AddressWithLines, + flatten::model::AddressLine, + flatten::model::StreetAddressLine, + flatten::model::CityAddressLine, + flatten::model::StateAddressLine + ]; +} + +###Mapping +Mapping flatten::mapping::H2Mapping +( + flatten::model::Person: Relational + { + ~primaryKey + ( + [flatten::store::H2DB]PERSON_SCHEMA.PERSON_TABLE.ID + ) + ~mainTable [flatten::store::H2DB]PERSON_SCHEMA.PERSON_TABLE + firstName: [flatten::store::H2DB]PERSON_SCHEMA.PERSON_TABLE.FIRSTNAME, + lastName: [flatten::store::H2DB]PERSON_SCHEMA.PERSON_TABLE.LASTNAME, + firm: Binding flatten::store::FirmBinding : [flatten::store::H2DB]PERSON_SCHEMA.PERSON_TABLE.FIRM_DETAILS, + manager[flatten_model_Person]: [flatten::store::H2DB]@manager + } +) + +###Runtime +Runtime flatten::runtime::H2Runtime +{ + mappings : + [ + flatten::mapping::H2Mapping + ]; + connections : + [ + flatten::store::H2DB : + [ + connection_1 : #{ + RelationalDatabaseConnection { + store: flatten::store::H2DB; + type: H2; + specification: LocalH2{ + testDataSetupSqls: [ + 'DROP SCHEMA IF EXISTS PERSON_SCHEMA CASCADE;', + 'CREATE SCHEMA PERSON_SCHEMA;', + 'CREATE TABLE PERSON_SCHEMA.PERSON_TABLE(ID INT PRIMARY KEY, FIRSTNAME VARCHAR(100), LASTNAME VARCHAR(100), FIRM_DETAILS VARCHAR(1000), MANAGERID INT);', + 'INSERT INTO PERSON_SCHEMA.PERSON_TABLE(ID,FIRSTNAME,LASTNAME,FIRM_DETAILS,MANAGERID) VALUES (1,\'Peter\',\'Smith\',\'{"legalName": "Firm X", "otherNames": ["O1", "O2"], "addresses": [{"name": "A1", "lines": [{"lineno": 1, "street": "S1"}, {"lineno": 2, "city": "C1"}]}, {"name": "A11", "lines": [{"lineno": 1, "street": "S1"}, {"lineno": 2, "city": "C2"}]}]}\',2);', + 'INSERT INTO PERSON_SCHEMA.PERSON_TABLE(ID,FIRSTNAME,LASTNAME,FIRM_DETAILS,MANAGERID) VALUES (2,\'John\',\'Johnson\',\'{"legalName": "Firm X", "otherNames": ["O1", "O2"], "addresses": [{"name": "A2", "lines": [{"lineno": 1, "street": "S2"}, {"lineno": 2, "city": "C2"}]}, {"name": "A22", "lines": [{"lineno": 1, "street": "S1"}, {"lineno": 2, "city": "C3"}]}]}\',4);', + 'INSERT INTO PERSON_SCHEMA.PERSON_TABLE(ID,FIRSTNAME,LASTNAME,FIRM_DETAILS,MANAGERID) VALUES (3,\'John\',\'Hill\',\'{"legalName": "Firm X", "otherNames": ["O1", "O2"], "addresses": [{"name": "A3", "lines": [{"lineno": 1, "street": "S3"}, {"lineno": 2, "city": "C1"}]}, {"name": "A32", "lines": [{"lineno": 1, "street": "S1"}, {"lineno": 2, "city": "C1"}]}]}\',2);', + 'INSERT INTO PERSON_SCHEMA.PERSON_TABLE(ID,FIRSTNAME,LASTNAME,FIRM_DETAILS,MANAGERID) VALUES (4,\'Anthony\',\'Allen\',\'{"legalName": "Firm X", "otherNames": ["O1", "O2"], "addresses": [{"name": "A4", "lines": [{"lineno": 1, "street": "S1"}, {"lineno": 2, "city": "C3"}]}]}\',null)', + 'INSERT INTO PERSON_SCHEMA.PERSON_TABLE(ID,FIRSTNAME,LASTNAME,FIRM_DETAILS,MANAGERID) VALUES (5,\'Fabrice\',\'Roberts\',\'{"legalName": "Firm A", "otherNames": ["O3", "O4"], "addresses": [{"name": "A5", "lines": [{"lineno": 1, "street": "S4"}, {"lineno": 2, "city": "C2"}]}, {"name": "A52", "lines": [{"lineno": 1, "street": "S1"}, {"lineno": 2, "city": "C4"}]}]}\',null)', + 'INSERT INTO PERSON_SCHEMA.PERSON_TABLE(ID,FIRSTNAME,LASTNAME,FIRM_DETAILS,MANAGERID) VALUES (6,\'Oliver\',\'Hill\',\'{"legalName": "Firm B", "otherNames": ["O5", "O6"], "addresses": [{"name": "A6", "lines": [{"lineno": 1, "street": "S5"}, {"lineno": 2, "city": "C4"}]}]}\',7)', + 'INSERT INTO PERSON_SCHEMA.PERSON_TABLE(ID,FIRSTNAME,LASTNAME,FIRM_DETAILS,MANAGERID) VALUES (7,\'David\',\'Harris\',\'{"legalName": "Firm B", "otherNames": ["O5", "O6"], "addresses": [{"name": "A7", "lines": [{"lineno": 1, "street": "S1"}, {"lineno": 2, "city": "C1"}]}]}\',null)' + ]; + }; + auth: Test; + } + }# + ] + ]; +} + + +###Pure +function flatten::semiStructuredPrimitivePropertyFlattening(): TabularDataSet[1] +{ + flatten::model::Person.all()->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.otherNames, 'Firm Other Name') + + ]) +} + +function flatten::semiStructuredComplexPropertyFlattening(): TabularDataSet[1] +{ + flatten::model::Person.all()->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.addresses.name, 'Firm Address Name') + ]) +} + +function flatten::semiStructuredPrimitivePropertyArrayIndexing(): TabularDataSet[1] +{ + flatten::model::Person.all()->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.otherNames->at(0), 'Firm Other Name 0'), + col(x | $x.firm.otherNames->at(2), 'Firm Other Name 2') + ]) +} + +function flatten::semiStructuredComplexPropertyArrayIndexing(): TabularDataSet[1] +{ + flatten::model::Person.all()->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.addresses->at(0).name, 'Firm Address 0 Name'), + col(x | $x.firm.addresses->at(2).name, 'Firm Address 2 Name') + ]) +} + +function flatten::semiStructuredComplexPropertyFlatteningFollowedBySubType(): TabularDataSet[1] +{ + flatten::model::Person.all()->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.addresses->subType(@flatten::model::AddressWithLines)->map(a | $a.lines->at(0).lineno), 'Firm Address Line 0 Line No') + ]) +} + +function flatten::semiStructuredComplexPropertyArrayIndexingFollowedBySubType(): TabularDataSet[1] +{ + flatten::model::Person.all()->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.addresses->at(0)->subType(@flatten::model::AddressWithLines).lines->at(0).lineno, 'Firm Address 0 Line 0 Line No') + ]) +} + +function flatten::semiStructuredPrimitivePropertyFiltering(): TabularDataSet[1] +{ + flatten::model::Person.all() + ->filter(x | $x.firm.otherNames->contains('A')) + ->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name') + ]) + ->distinct() +} + +function flatten::semiStructuredPrimitivePropertyFilteringInProject(): TabularDataSet[1] +{ + flatten::model::Person.all()->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.otherNames->filter(x | $x->startsWith('A')), 'Firm Other Name') + ]) +} + +function flatten::semiStructuredComplexPropertyFiltering(): TabularDataSet[1] +{ + flatten::model::Person.all() + ->filter(p | $p.firm.addresses->exists(a | $a.name == 'B')) + ->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name') + ]) +} + +function flatten::semiStructuredComplexPropertyFilteringInProject(): TabularDataSet[1] +{ + flatten::model::Person.all() + ->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.addresses->filter(a | $a.name == 'A').name, 'Firm Address Name 1'), + col(x | $x.firm.addresses->filter(a | $a.name == 'B').name, 'Firm Address Name 2') + ]) +} + +function flatten::semiStructuredComplexPropertyFilteringInProjectFollowedBySubType(): TabularDataSet[1] +{ + flatten::model::Person.all() + ->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.addresses->filter(a | $a.name == 'A')->subType(@flatten::model::AddressWithLines).lines->at(0).lineno, 'Firm Address Name 1'), + col(x | $x.firm.addresses->filter(a | $a.name == 'B').name, 'Firm Address Name 2') + ]) +} + +function flatten::semiStructuredMultiLevelFlattening(): TabularDataSet[1] +{ + flatten::model::Person.all() + ->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.addresses->filter(a | $a.name == 'A')->subType(@flatten::model::AddressWithLines).lines.lineno, 'Firm Address Name Line No') + ]) +} + +function flatten::semiStructuredSubAggregation(): TabularDataSet[1] +{ + flatten::model::Person.all() + ->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.addresses.name->joinStrings(';'), 'Firm Address Names') + ]) +} + +function flatten::semiStructuredSubAggregationDeep(): TabularDataSet[1] +{ + flatten::model::Person.all() + ->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.legalName, 'Firm Name'), + col(x | $x.firm.addresses->subType(@flatten::model::AddressWithLines).lines.lineno->sum(), 'Firm Address Line No Sum') + ]) +} + +function flatten::semiStructuredMultiFlatten(): TabularDataSet[1] +{ + flatten::model::Person.all() + ->project([ + col(x | $x.firstName, 'First Name'), + col(x | $x.firm.addresses.name, 'Firm Address Name'), + col(x | $x.firm.addresses->subType(@flatten::model::AddressWithLines).lines->at(0).lineno, 'Firm Address Line 0 No'), + col(x | $x.firm.otherNames, 'Firm Other Name') + ]) +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions_1_4_200.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions_1_4_200.java index 17038e00a4d..22e2a1d1101 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions_1_4_200.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions_1_4_200.java @@ -20,7 +20,9 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.codec.digest.DigestUtils; import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.h2.tools.SimpleResultSet; import org.h2.value.Value; +import org.h2.value.ValueArray; import org.h2.value.ValueBoolean; import org.h2.value.ValueDouble; import org.h2.value.ValueFloat; @@ -29,9 +31,15 @@ import org.h2.value.ValueNull; import org.h2.value.ValueString; +import java.math.BigDecimal; import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Types; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -158,4 +166,159 @@ public static String legend_h2_extension_split_part(String string, String token, return parts.length > readjustedPart ? parts[readjustedPart] : null; } + + + private static HashSet extractProperty(HashSet resultSet, Object pathToExtract) + { + HashSet res = new HashSet<>(); + if (pathToExtract instanceof String) + { + String property = (String) pathToExtract; + if (property.equals("*")) + { + for (Object r: resultSet) + { + if (!(r instanceof Iterable)) + { + continue; + } + for (Object o : (Iterable) r) + { + res.add(o); + } + } + } + else + { + for (Object r: resultSet) + { + try + { + Object o = ((HashMap)(r)).get(property); + if (o != null) + { + res.add(o); + } + } + catch (Exception e) + { + e.printStackTrace(); // don't stop execution + } + } + } + } + else + { + int index = (int) pathToExtract; + for (Object r: resultSet) + { + try + { + res.add(((ArrayList)(r)).get(index)); + } + catch (Exception e) + { + e.printStackTrace(); + } + + } + } + return res; + } + + public static ResultSet legend_h2_extension_flatten_array(Connection conn, String tableName, String toFlattenColumnName, ValueArray jsonPaths) + { + try + { + String sql = String.format("select distinct %s as flat from %s", toFlattenColumnName, tableName); + ResultSet resultSet = conn.createStatement().executeQuery(sql); + HashSet toFlatten = new HashSet<>(); + while (resultSet.next()) + { + String json = resultSet.getString("flat"); + toFlatten.add(OBJECT_MAPPER.readValue(json, HashMap.class)); + } + + ArrayList pathsToExtract = OBJECT_MAPPER.readValue(jsonPaths.getString(), ArrayList.class); + + for (Object path: pathsToExtract) + { + toFlatten = extractProperty(toFlatten, path); + } + + SimpleResultSet flattenedResultSet = new SimpleResultSet(); + + flattenedResultSet.addColumn("__INPUT__", Types.VARCHAR, 1000, 0); // using the original array as joinKey + + // use first non-null object to infer the type of value + boolean resolvedFlattenedType = false; + + for (Object o: toFlatten) + { + if (!(o instanceof Iterable)) + { + continue; + } + for (Object value : (Iterable) o) + { + if (value instanceof Map || value instanceof List || value instanceof String) + { + flattenedResultSet.addColumn("VALUE", Types.VARCHAR, 1000, 0); + } + else if (value instanceof Boolean) + { + flattenedResultSet.addColumn("VALUE", Types.BOOLEAN, 0, 0); + } + else if (value instanceof Double || value instanceof Float || value instanceof BigDecimal) + { + flattenedResultSet.addColumn("VALUE", Types.DOUBLE, 20, 20); + } + else if (value instanceof Integer || value instanceof Long) + { + flattenedResultSet.addColumn("VALUE", Types.BIGINT, 0, 0); + } + else + { + throw new RuntimeException("unsupported data type in h2 extension"); + } + resolvedFlattenedType = true; + break; + } + if (resolvedFlattenedType) + { + break; + } + } + + if (!resolvedFlattenedType) + { + flattenedResultSet.addColumn("VALUE", Types.VARCHAR, 1000, 0); + return flattenedResultSet; + } + + for (Object o: toFlatten) + { + if (!(o instanceof Iterable)) + { + continue; + } + for (Object value : (Iterable) o) + { + if (value instanceof Map || value instanceof List) + { + flattenedResultSet.addRow(Arrays.asList(OBJECT_MAPPER.writeValueAsString(o), OBJECT_MAPPER.writeValueAsString(value)).toArray()); + } + else + { + flattenedResultSet.addRow(Arrays.asList(OBJECT_MAPPER.writeValueAsString(o), value).toArray()); + } + } + } + return flattenedResultSet; + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure index ce76a5a6195..d4497a411cd 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure @@ -7378,6 +7378,24 @@ function meta::relational::functions::pureToSqlQuery::extractTableAliasColumns(e ); } +/* +* Given a relationalElement and columnName c0, Returns the root Table and the columnName in the root table for the column +*/ +function meta::relational::functions::pureToSqlQuery::findTableForColumnInAlias(z:RelationalOperationElement[1], columnName:String[0..1]):Pair[1] +{ + $z->match([ + s:SelectSQLQuery[1] | let tac = $s.columns->filter(c|$c->extractColumnName() == $columnName->toOne())->extractTableAliasColumns()->toOne(); $tac->findTableForColumnInAlias($columnName);, + a:Alias[1] | $a.relationalElement->findTableForColumnInAlias($columnName), + s:SemiStructuredObjectNavigation[1]| $s.operand->findTableForColumnInAlias($columnName), + s:SemiStructuredArrayFlatten[1]|$s.navigation->findTableForColumnInAlias($columnName), + s:SemiStructuredArrayFlattenOutput[1]|$s.tableAliasColumn->findTableForColumnInAlias($columnName), + r:RelationalOperationElementWithJoin[1] | let tac = $r.relationalOperationElement->extractTableAliasColumns()->toOne(); $tac->findTableForColumnInAlias($columnName);, + tac:TableAliasColumn[1] | $tac.alias.relationalElement->findTableForColumnInAlias($tac.column.name);, // ignore passed columnName + v:View[1] | let rop = $v.columnMappings->filter(c|$c.columnName == $columnName->toOne())->toOne(); $rop.relationalOperationElement->findTableForColumnInAlias([]);, + t:Table[1] | assert($t.columns->cast(@Column).name->contains($columnName->toOne()), 'can\'t find ' + $columnName->toOne() + ' in Table: ' + $t.schema.name + '.' + $t.name); pair($t, $columnName->toOne()); + ]); +} + function meta::relational::functions::pureToSqlQuery::getJoinTreeNode(relationalPropertyMapping:RelationalPropertyMapping[1]):JoinTreeNode[0..1] { $relationalPropertyMapping.relationalOperationElement->match([r:RelationalOperationElementWithJoin[1] | $r.joinTreeNode, a:Any[*] | []]); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension1_4_200.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension1_4_200.pure index 03f6056e828..c04542641ed 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension1_4_200.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension1_4_200.pure @@ -1,3 +1,6 @@ +import meta::relational::metamodel::relation::*; +import meta::pure::extension::*; +import meta::relational::metamodel::join::*; import meta::relational::functions::sqlQueryToString::default::*; import meta::external::store::relational::runtime::*; import meta::relational::runtime::*; @@ -20,6 +23,7 @@ function meta::relational::functions::sqlQueryToString::h2::v1_4_200::createDbEx isDbReservedIdentifier = {str:String[1]| $str->toLower()->in($reservedWords)}, literalProcessor = $literalProcessor, windowColumnProcessor = processWindowColumn_WindowColumn_1__SqlGenerationContext_1__String_1_, + lateralJoinProcessor = processJoinTreeNodeWithLateralJoinForH2_JoinTreeNode_1__DbConfig_1__Format_1__Extension_MANY__String_1_, semiStructuredElementProcessor = processSemiStructuredElementForH2_RelationalOperationElement_1__SqlGenerationContext_1__String_1_, joinStringsProcessor = processJoinStringsOperationForH2_JoinStrings_1__SqlGenerationContext_1__String_1_, selectSQLQueryProcessor = processSelectSQLQueryDefault_SelectSQLQuery_1__SqlGenerationContext_1__Boolean_1__String_1_, @@ -149,6 +153,24 @@ function <> meta::relational::functions::sqlQueryToString::h2::v 'merge into ' + $upsertQuery.data.name + ' (' + $columnNames + ') values (' + $literalValues + ')'; } +function <> meta::relational::functions::sqlQueryToString::h2::v1_4_200::processJoinTreeNodeWithLateralJoinForH2(j:JoinTreeNode[1], dbConfig : DbConfig[1], format:Format[1], extensions:Extension[*]):String[1] +{ + // keeping consistent with snowflake + assert(processOperation($j.join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) == '1 = 1', | 'Filter in column projections is not supported. Use a Post Filter if filtering is necessary'); + + assert($j.alias.relationalElement->instanceOf(SemiStructuredArrayFlatten), | 'Lateral join in H2 should be followed by flatten operation'); + + let lhs = ^TableAliasColumn(column = ^Column(name = '__INPUT__', type = ^meta::relational::metamodel::datatype::SemiStructured()),alias = $j.alias); + let rhs = $j.alias.relationalElement->cast(@SemiStructuredArrayFlatten).navigation->cast(@SemiStructuredObjectNavigation); + let joinOperation = ^DynaFunction(name= 'equal', parameters = [$lhs, ^$rhs(returnType=String)]); + + ' ' + $format.separator() + 'left outer join ' + + $j.alias + ->map(a|^$a(name = '"' + $a.name + '"')) + ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() + + ' ' + 'on (' + processOperation($joinOperation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')'; +} + function <> meta::relational::functions::sqlQueryToString::h2::v1_4_200::processExtractFromSemiStructuredParamsForH2(params:String[3]):String[1] { let baseRelationalOp = $params->at(0); @@ -187,10 +209,69 @@ function <> meta::relational::functions::sqlQueryToString::h2::v function <> meta::relational::functions::sqlQueryToString::h2::v1_4_200::processSemiStructuredElementForH2(s:RelationalOperationElement[1], sgc:SqlGenerationContext[1]): String[1] { $s->match([ - o:SemiStructuredObjectNavigation[1] | $o->processSemiStructuredObjectNavigationForH2($sgc) + o:SemiStructuredObjectNavigation[1] | $o->processSemiStructuredObjectNavigationForH2($sgc), + a:SemiStructuredArrayFlatten[1] | $a->processSemiStructuredArrayFlattenForH2($sgc), + a:SemiStructuredArrayFlattenOutput[1] | $a->processSemiStructuredArrayFlattenOutputForH2($sgc) ]) } +/* +* returns property accesses to extract the semi structured property starting from root +*/ +function <> meta::relational::functions::sqlQueryToString::h2::v1_4_200::propertyAccessForSemiStructuredObjectNavigationH2(z:SemiStructuredObjectNavigation[1], sgc:SqlGenerationContext[1]): String[*] +{ + let elementAccess = $z->match([ + p: SemiStructuredPropertyAccess[1] | + let propertyAccess = '"' + $p.property->cast(@Literal).value->cast(@String) + '"'; + if ($p.index->isNotEmpty(), + | $propertyAccess->concatenate($p.index->toOne()->cast(@Literal).value->toString()), + | $propertyAccess + );, + a: SemiStructuredArrayElementAccess[1] | $a.index->toOne()->cast(@Literal).value->toString() + ]); + $z.operand->match([ + s: SemiStructuredObjectNavigation[1] | $s->propertyAccessForSemiStructuredObjectNavigationH2($sgc), + a: SemiStructuredArrayFlatten[1] | $a.navigation->cast(@SemiStructuredObjectNavigation)->propertyAccessForSemiStructuredObjectNavigationH2($sgc)->concatenate('"*"'), + s: SemiStructuredArrayFlattenOutput[1] | let flattening = $s.tableAliasColumn.alias.relationalElement->cast(@SemiStructuredArrayFlatten); + $flattening.navigation->cast(@SemiStructuredObjectNavigation)->propertyAccessForSemiStructuredObjectNavigationH2($sgc)->concatenate('"*"');, + a: Any[1] | [] + ])->concatenate($elementAccess); +} + +function <> meta::relational::functions::sqlQueryToString::h2::v1_4_200::processSemiStructuredArrayFlattenForH2(s:SemiStructuredArrayFlatten[1], sgc:SqlGenerationContext[1]): String[1] +{ + let rootTableAndColumnName = $s->meta::relational::functions::pureToSqlQuery::findTableForColumnInAlias([]); + + let jsonPaths = $s.navigation->match([ // assumes input to ssaf is always sson + s: SemiStructuredObjectNavigation[1] | $s->propertyAccessForSemiStructuredObjectNavigationH2($sgc) + ]); + + let schema = $rootTableAndColumnName.first.schema.name; + + let processedNavigation = $s.navigation->processOperation($sgc); + 'legend_h2_extension_flatten_array(' + '\'' + $rootTableAndColumnName.first->processOperation($sgc) + '\',\'' + $rootTableAndColumnName.second->processColumnName($sgc.dbConfig) + '\',ARRAY[\'' + $jsonPaths->joinStrings('\',\'') + '\'])'; +} + +function <> meta::relational::functions::sqlQueryToString::h2::v1_4_200::processSemiStructuredArrayFlattenOutputForH2(s:SemiStructuredArrayFlattenOutput[1], sgc:SqlGenerationContext[1]): String[1] +{ + let doubleQuote = if($sgc.config.useQuotesForTableAliasColumn == false, |'', |'"'); + let processedIdentifier = $sgc.dbConfig.identifierProcessor($doubleQuote + $s.tableAliasColumn.alias.name->toOne() + $doubleQuote); + let elementAccess = $processedIdentifier + '.' + processColumnName('VALUE', $sgc.dbConfig); + $elementAccess->castToReturnTypeForSemiStructuredData($s.returnType); +} + +function <> meta::relational::functions::sqlQueryToString::h2::v1_4_200::castToReturnTypeForSemiStructuredData(elementAccess:String[1], returnType:Type[0..1]): String[1] +{ + if ($returnType == String, | 'cast(' + $elementAccess + ' as varchar)', | + if ($returnType == Boolean, | 'cast(' + $elementAccess + ' as boolean)', | + if ($returnType == Float, | 'cast(' + $elementAccess + ' as float)', | + if ($returnType == Integer, | 'cast(' + $elementAccess + ' as integer)', | + if ($returnType == StrictDate, | 'cast(' + $elementAccess + ' as date)', | + if ($returnType->isNotEmpty() && $returnType->toOne()->_subTypeOf(Date), | 'cast(' + $elementAccess + ' as timestamp)', | + if ($returnType->isNotEmpty() && $returnType->toOne()->instanceOf(Enumeration), | 'cast(' + $elementAccess + ' as varchar)', | + $elementAccess))))))); +} + function <> meta::relational::functions::sqlQueryToString::h2::v1_4_200::processSemiStructuredObjectNavigationForH2(s:SemiStructuredObjectNavigation[1], sgc:SqlGenerationContext[1]): String[1] { // Use a user defined function for H2 (testing purpose) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure index 0e9124cb439..6401750cfd8 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure @@ -1,3 +1,6 @@ +import meta::relational::functions::pureToSqlQuery::*; +import meta::relational::metamodel::relation::*; +import meta::pure::extension::*; import meta::relational::functions::pureToSqlQuery::metamodel::*; import meta::relational::functions::sqlQueryToString::default::*; import meta::external::store::relational::runtime::*; @@ -24,6 +27,7 @@ function meta::relational::functions::sqlQueryToString::h2::v2_1_214::createDbEx isDbReservedIdentifier = {str:String[1]| $str->toLower()->in($reservedWords)}, literalProcessor = $literalProcessor, windowColumnProcessor = processWindowColumn_WindowColumn_1__SqlGenerationContext_1__String_1_, + lateralJoinProcessor = processJoinTreeNodeWithLateralJoinForH2_JoinTreeNode_1__DbConfig_1__Format_1__Extension_MANY__String_1_, semiStructuredElementProcessor = processSemiStructuredElementForH2_RelationalOperationElement_1__SqlGenerationContext_1__String_1_, joinStringsProcessor = processJoinStringsOperationForH2_JoinStrings_1__SqlGenerationContext_1__String_1_, selectSQLQueryProcessor = processSelectSQLQueryForH2_SelectSQLQuery_1__SqlGenerationContext_1__Boolean_1__String_1_, @@ -262,6 +266,24 @@ function <> meta::relational::functions::sqlQueryToString::h2::v 'merge into ' + $upsertQuery.data.name + ' (' + $columnNames + ') values (' + $literalValues + ')'; } +function <> meta::relational::functions::sqlQueryToString::h2::v2_1_214::processJoinTreeNodeWithLateralJoinForH2(j:JoinTreeNode[1], dbConfig : DbConfig[1], format:Format[1], extensions:Extension[*]):String[1] +{ + // keeping consistent with snowflake + assert(processOperation($j.join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) == '1 = 1', | 'Filter in column projections is not supported. Use a Post Filter if filtering is necessary'); + + assert($j.alias.relationalElement->instanceOf(SemiStructuredArrayFlatten), | 'Lateral join in H2 should be followed by flatten operation'); + + let lhs = ^TableAliasColumn(column = ^Column(name = '__INPUT__', type = ^meta::relational::metamodel::datatype::SemiStructured()),alias = $j.alias); + let rhs = $j.alias.relationalElement->cast(@SemiStructuredArrayFlatten).navigation->cast(@SemiStructuredObjectNavigation); + let joinOperation = ^DynaFunction(name= 'equal', parameters = [$lhs, ^$rhs(returnType=String)]); + + ' ' + $format.separator() + 'left outer join ' + + $j.alias + ->map(a|^$a(name = '"' + $a.name + '"')) + ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() + + ' ' + 'on (' + processOperation($joinOperation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')'; +} + function <> meta::relational::functions::sqlQueryToString::h2::v2_1_214::processExtractFromSemiStructuredParamsForH2(params:String[3]):String[1] { let baseRelationalOp = $params->at(0); @@ -300,10 +322,69 @@ function <> meta::relational::functions::sqlQueryToString::h2::v function <> meta::relational::functions::sqlQueryToString::h2::v2_1_214::processSemiStructuredElementForH2(s:RelationalOperationElement[1], sgc:SqlGenerationContext[1]): String[1] { $s->match([ - o:SemiStructuredObjectNavigation[1] | $o->processSemiStructuredObjectNavigationForH2($sgc) + o:SemiStructuredObjectNavigation[1] | $o->processSemiStructuredObjectNavigationForH2($sgc), + a:SemiStructuredArrayFlatten[1] | $a->processSemiStructuredArrayFlattenForH2($sgc), + a:SemiStructuredArrayFlattenOutput[1] | $a->processSemiStructuredArrayFlattenOutputForH2($sgc) ]) } +/* +* returns property accesses to extract the semi structured property starting from root +*/ +function <> meta::relational::functions::sqlQueryToString::h2::v2_1_214::propertyAccessForSemiStructuredObjectNavigationH2(z:SemiStructuredObjectNavigation[1], sgc:SqlGenerationContext[1]): String[*] +{ + let elementAccess = $z->match([ + p: SemiStructuredPropertyAccess[1] | + let propertyAccess = '"' + $p.property->cast(@Literal).value->cast(@String) + '"'; + if ($p.index->isNotEmpty(), + | $propertyAccess->concatenate($p.index->toOne()->cast(@Literal).value->toString()), + | $propertyAccess + );, + a: SemiStructuredArrayElementAccess[1] | $a.index->toOne()->cast(@Literal).value->toString() + ]); + $z.operand->match([ + s: SemiStructuredObjectNavigation[1] | $s->propertyAccessForSemiStructuredObjectNavigationH2($sgc), + a: SemiStructuredArrayFlatten[1] | $a.navigation->cast(@SemiStructuredObjectNavigation)->propertyAccessForSemiStructuredObjectNavigationH2($sgc)->concatenate('"*"'), + s: SemiStructuredArrayFlattenOutput[1] | let flattening = $s.tableAliasColumn.alias.relationalElement->cast(@SemiStructuredArrayFlatten); + $flattening.navigation->cast(@SemiStructuredObjectNavigation)->propertyAccessForSemiStructuredObjectNavigationH2($sgc)->concatenate('"*"');, + a: Any[1] | [] + ])->concatenate($elementAccess); +} + +function <> meta::relational::functions::sqlQueryToString::h2::v2_1_214::processSemiStructuredArrayFlattenForH2(s:SemiStructuredArrayFlatten[1], sgc:SqlGenerationContext[1]): String[1] +{ + let rootTableAndColumnName = $s->meta::relational::functions::pureToSqlQuery::findTableForColumnInAlias([]); + + let jsonPaths = $s.navigation->match([ // assumes input to ssaf is always sson + s: SemiStructuredObjectNavigation[1] | $s->propertyAccessForSemiStructuredObjectNavigationH2($sgc) + ]); + + let schema = $rootTableAndColumnName.first.schema.name; + + let processedNavigation = $s.navigation->processOperation($sgc); + 'legend_h2_extension_flatten_array(' + '\'' + $rootTableAndColumnName.first->processOperation($sgc) + '\',\'' + $rootTableAndColumnName.second->processColumnName($sgc.dbConfig) + '\',ARRAY[\'' + $jsonPaths->joinStrings('\',\'') + '\'])'; +} + +function <> meta::relational::functions::sqlQueryToString::h2::v2_1_214::processSemiStructuredArrayFlattenOutputForH2(s:SemiStructuredArrayFlattenOutput[1], sgc:SqlGenerationContext[1]): String[1] +{ + let doubleQuote = if($sgc.config.useQuotesForTableAliasColumn == false, |'', |'"'); + let processedIdentifier = $sgc.dbConfig.identifierProcessor($doubleQuote + $s.tableAliasColumn.alias.name->toOne() + $doubleQuote); + let elementAccess = $processedIdentifier + '.' + processColumnName('VALUE', $sgc.dbConfig); + $elementAccess->castToReturnTypeForSemiStructuredData($s.returnType); +} + +function <> meta::relational::functions::sqlQueryToString::h2::v2_1_214::castToReturnTypeForSemiStructuredData(elementAccess:String[1], returnType:Type[0..1]): String[1] +{ + if ($returnType == String, | 'cast(' + $elementAccess + ' as varchar)', | + if ($returnType == Boolean, | 'cast(' + $elementAccess + ' as boolean)', | + if ($returnType == Float, | 'cast(' + $elementAccess + ' as float)', | + if ($returnType == Integer, | 'cast(' + $elementAccess + ' as integer)', | + if ($returnType == StrictDate, | 'cast(' + $elementAccess + ' as date)', | + if ($returnType->isNotEmpty() && $returnType->toOne()->_subTypeOf(Date), | 'cast(' + $elementAccess + ' as timestamp)', | + if ($returnType->isNotEmpty() && $returnType->toOne()->instanceOf(Enumeration), | 'cast(' + $elementAccess + ' as varchar)', | + $elementAccess))))))); +} + function <> meta::relational::functions::sqlQueryToString::h2::v2_1_214::processSemiStructuredObjectNavigationForH2(s:SemiStructuredObjectNavigation[1], sgc:SqlGenerationContext[1]): String[1] { // Use a user defined function for H2 (testing purpose) @@ -323,14 +404,7 @@ function <> meta::relational::functions::sqlQueryToString::h2::v a: SemiStructuredArrayElementAccess[1] | semiStructuredArrayElementAccessForH2($processedOperand, $a.index->cast(@Literal).value->toString()) ]); - if ($s.returnType == String, | 'cast(' + $elementAccess + ' as varchar)', | - if ($s.returnType == Boolean, | 'cast(' + $elementAccess + ' as boolean)', | - if ($s.returnType == Float, | 'cast(' + $elementAccess + ' as float)', | - if ($s.returnType == Integer, | 'cast(' + $elementAccess + ' as integer)', | - if ($s.returnType == StrictDate, | 'cast(' + $elementAccess + ' as date)', | - if ($s.returnType->isNotEmpty() && $s.returnType->toOne()->_subTypeOf(Date), | 'cast(' + $elementAccess + ' as timestamp)', | - if ($s.returnType->isNotEmpty() && $s.returnType->toOne()->instanceOf(Enumeration), | 'cast(' + $elementAccess + ' as varchar)', | - $elementAccess))))))); + $elementAccess->castToReturnTypeForSemiStructuredData($s.returnType); } function <> meta::relational::functions::sqlQueryToString::h2::v2_1_214::processJoinStringsOperationForH2(js:JoinStrings[1], sgc:SqlGenerationContext[1]): String[1] @@ -614,18 +688,20 @@ function <> meta::relational::functions::sqlQueryToString::h2::v ->processOperation($dbConfig, $format->indent(), $extensions), j:JoinTreeNode[1] | if($j.joinType == JoinType.FULL_OUTER, - | - // This should have been converted earlier to avoid a FULL_OUTER reaching this point + | + // This should have been converted earlier to avoid a FULL_OUTER reaching this point fail($j.joinType->toOne()->toString() + ' join not supported in H2'); ''; , + | + if($j.lateral == true, + | $dbConfig.lateralJoinProcessor($j, $dbConfig, $format, $extensions), | $j.joinType->map(jt|$jt->meta::relational::functions::sqlQueryToString::default::processJoinType($dbConfig, $format, $extensions))->orElse('') - + if ($j.lateral == true, | 'lateral ', | '') + $j.alias ->map(a|^$a(name = '"' + $a.name + '"')) //Not sure why this is necessary, but it's retained to keep the generated SQL the same as previously (and does no real harm) ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() + ' ' + 'on (' + $j.join.operation->wrapH2Boolean($extensions)->processOperation($dbConfig, $format->indent(), ^Config(), $extensions) + ')'; - ), + )), a:Any[1] | '' ] );