Skip to content

Commit

Permalink
Fix to infer sourceDataType in cross store joins (#2557)
Browse files Browse the repository at this point in the history
  • Loading branch information
@theodosis1989
theodosis1989 authored Mar 28, 2024
1 parent 774db2e commit db9e8ea
Show file tree
Hide file tree
Showing 24 changed files with 231 additions and 180 deletions.
8 changes: 4 additions & 4 deletions ...n/execution/stores/relational/test/semiStructured/TestSnowflakeExplodeSemiStructured.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,12 @@ public void testComplexProjectFlattenedAndExplodedPropertiesInProject()
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Id, String, VARCHAR(100), \"\"), (Entity Tag, String, \"\", \"\"), (Entity Tag Id, String, \"\", \"\"), (Orders/Id, String, VARCHAR(100), \"\")]\n" +
" type = TDS[(Id, String, VARCHAR(100), \"\"), (Entity Tag, String, VARCHAR(65536), \"\"), (Entity Tag Id, String, VARCHAR(65536), \"\"), (Orders/Id, String, VARCHAR(100), \"\")]\n" +
" resultColumns = [(\"Id\", VARCHAR(100)), (\"Entity Tag\", \"\"), (\"Entity Tag Id\", \"\"), (\"Orders/Id\", VARCHAR(100))]\n" +
" sql = select \"root\".ID as \"Id\", \"ss_flatten_0\".VALUE['tag']::varchar as \"Entity Tag\", \"ss_flatten_0\".VALUE['tagId']::varchar as \"Entity Tag Id\", \"blocks_1\".ID as \"Orders/Id\" from Semistructured.Blocks as \"root\" inner join lateral flatten(input => \"root\".BLOCKDATA['relatedEntities'], outer => true, recursive => false, mode => 'array') as \"ss_flatten_0\" left outer join (select \"orders_0\".ID, \"orders_0\".IDENTIFIER, \"orders_0\".QUANTITY, \"orders_0\".SIDE, \"orders_0\".PRICE, \"blocks_2\".leftJoinKey_0 as leftJoinKey_0 from (select \"ss_flatten_1\".VALUE as flattened_prop, \"root\".ID as leftJoinKey_0 from Semistructured.Blocks as \"root\" inner join lateral flatten(input => \"root\".BLOCKDATA['relatedEntities'], outer => true, recursive => false, mode => 'array') as \"ss_flatten_1\") as \"blocks_2\" inner join Semistructured.Orders as \"orders_0\" on (to_varchar(get_path(\"blocks_2\".flattened_prop, 'tag')) = 'order' and to_varchar(get_path(\"blocks_2\".flattened_prop, 'tagId')) = \"orders_0\".ID)) as \"blocks_1\" on (\"root\".ID = \"blocks_1\".leftJoinKey_0)\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Id, String, VARCHAR(100), \"\"), (Entity Tag, String, \"\", \"\"), (Entity Tag Id, String, \"\", \"\"), (Orders/Id, String, VARCHAR(100), \"\")]\n";
String TDSType = " type = TDS[(Id, String, VARCHAR(100), \"\"), (Entity Tag, String, VARCHAR(65536), \"\"), (Entity Tag Id, String, VARCHAR(65536), \"\"), (Orders/Id, String, VARCHAR(100), \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}

Expand Down Expand Up @@ -193,12 +193,12 @@ public void testAggregationAggregateExplodedPropertyInsideProject()
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Id, String, VARCHAR(100), \"\"), (Account, String, VARCHAR(100), \"\"), (Buy Order, Integer, \"\", \"\")]\n" +
" type = TDS[(Id, String, VARCHAR(100), \"\"), (Account, String, VARCHAR(100), \"\"), (Buy Order, Integer, BIGINT, \"\")]\n" +
" resultColumns = [(\"Id\", VARCHAR(100)), (\"Account\", VARCHAR(100)), (\"Buy Order\", INT)]\n" +
" sql = select \"root\".ID as \"Id\", \"root\".ACCOUNT as \"Account\", \"blocks_1\".aggCol as \"Buy Order\" from Semistructured.Blocks as \"root\" left outer join (select \"blocks_2\".ID as ID, sum(\"blocks_3\".QUANTITY) as aggCol from Semistructured.Blocks as \"blocks_2\" left outer join (select \"orders_0\".ID, \"orders_0\".IDENTIFIER, \"orders_0\".QUANTITY, \"orders_0\".SIDE, \"orders_0\".PRICE, \"blocks_4\".leftJoinKey_0 as leftJoinKey_0 from (select \"ss_flatten_0\".VALUE as flattened_prop, \"root\".ID as leftJoinKey_0 from Semistructured.Blocks as \"root\" inner join lateral flatten(input => \"root\".BLOCKDATA['relatedEntities'], outer => true, recursive => false, mode => 'array') as \"ss_flatten_0\") as \"blocks_4\" inner join Semistructured.Orders as \"orders_0\" on (to_varchar(get_path(\"blocks_4\".flattened_prop, 'tag')) = 'order' and to_varchar(get_path(\"blocks_4\".flattened_prop, 'tagId')) = \"orders_0\".ID)) as \"blocks_3\" on (\"blocks_2\".ID = \"blocks_3\".leftJoinKey_0) where \"blocks_3\".SIDE = 'BUY' group by \"blocks_2\".ID) as \"blocks_1\" on (\"root\".ID = \"blocks_1\".ID)\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Id, String, VARCHAR(100), \"\"), (Account, String, VARCHAR(100), \"\"), (Buy Order, Integer, \"\", \"\")]\n";
String TDSType = " type = TDS[(Id, String, VARCHAR(100), \"\"), (Account, String, VARCHAR(100), \"\"), (Buy Order, Integer, BIGINT, \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}

Expand Down
4 changes: 2 additions & 2 deletions ...ion/stores/relational/test/semiStructured/TestSnowflakeExtractFromSemiStructuredJoin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,12 @@ public void testJoinOnSemiStructuredProperty()
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(First Name, String, VARCHAR(100), \"\"), (Last Name, String, VARCHAR(100), \"\"), (Firm/Legal Name, String, \"\", \"\")]\n" +
" type = TDS[(First Name, String, VARCHAR(100), \"\"), (Last Name, String, VARCHAR(100), \"\"), (Firm/Legal Name, String, VARCHAR(65536), \"\")]\n" +
" resultColumns = [(\"First Name\", VARCHAR(100)), (\"Last Name\", VARCHAR(100)), (\"Firm/Legal Name\", \"\")]\n" +
" sql = select \"root\".FIRSTNAME as \"First Name\", \"root\".LASTNAME as \"Last Name\", \"firm_table_0\".FIRM_DETAILS['legalName']::varchar as \"Firm/Legal Name\" from PERSON_SCHEMA.PERSON_TABLE as \"root\" left outer join FIRM_SCHEMA.FIRM_TABLE as \"firm_table_0\" on (to_number(get_path(\"root\".FIRM, 'ID')) = to_number(get_path(\"firm_table_0\".FIRM_DETAILS, 'ID')))\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(First Name, String, VARCHAR(100), \"\"), (Last Name, String, VARCHAR(100), \"\"), (Firm/Legal Name, String, \"\", \"\")]\n";
String TDSType = " type = TDS[(First Name, String, VARCHAR(100), \"\"), (Last Name, String, VARCHAR(100), \"\"), (Firm/Legal Name, String, VARCHAR(65536), \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}

Expand Down
12 changes: 6 additions & 6 deletions ...n/stores/relational/test/semiStructured/TestSnowflakeExtractFromSemiStructuredSimple.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@ public void testDotAndBracketNotationAccess()
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Id, Integer, INT, \"\"), (Dot Only, StrictDate, \"\", \"\"), (Bracket Only, DateTime, \"\", \"\"), (Dot & Bracket, String, \"\", \"\")]\n" +
" type = TDS[(Id, Integer, INT, \"\"), (Dot Only, StrictDate, DATE, \"\"), (Bracket Only, DateTime, TIMESTAMP, \"\"), (Dot & Bracket, String, VARCHAR(65536), \"\")]\n" +
" resultColumns = [(\"Id\", INT), (\"Dot Only\", \"\"), (\"Bracket Only\", \"\"), (\"Dot & Bracket\", \"\")]\n" +
" sql = select \"root\".ID as \"Id\", to_date(get_path(\"root\".FIRM_DETAILS, 'dates.estDate')) as \"Dot Only\", to_timestamp(get_path(\"root\".FIRM_DETAILS, '[\"dates\"][\"last Update\"]')) as \"Bracket Only\", to_varchar(get_path(\"root\".FIRM_DETAILS, 'address.lines[1][\"details\"]')) as \"Dot & Bracket\" from FIRM_SCHEMA.FIRM_TABLE as \"root\"\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Id, Integer, INT, \"\"), (Dot Only, StrictDate, \"\", \"\"), (Bracket Only, DateTime, \"\", \"\"), (Dot & Bracket, String, \"\", \"\")]\n";
String TDSType = " type = TDS[(Id, Integer, INT, \"\"), (Dot Only, StrictDate, DATE, \"\"), (Bracket Only, DateTime, TIMESTAMP, \"\"), (Dot & Bracket, String, VARCHAR(65536), \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}

Expand All @@ -47,12 +47,12 @@ public void testArrayElementNoFlattenAccess()
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Id, Integer, INT, \"\"), (Second Line of Address, String, \"\", \"\")]\n" +
" type = TDS[(Id, Integer, INT, \"\"), (Second Line of Address, String, VARCHAR(65536), \"\")]\n" +
" resultColumns = [(\"Id\", INT), (\"Second Line of Address\", \"\")]\n" +
" sql = select \"root\".ID as \"Id\", to_varchar(get_path(\"root\".FIRM_DETAILS, 'address.lines[1][\"details\"]')) as \"Second Line of Address\" from FIRM_SCHEMA.FIRM_TABLE as \"root\"\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Id, Integer, INT, \"\"), (Second Line of Address, String, \"\", \"\")]\n";
String TDSType = " type = TDS[(Id, Integer, INT, \"\"), (Second Line of Address, String, VARCHAR(65536), \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}

Expand Down Expand Up @@ -81,12 +81,12 @@ public void testAllDataTypesAccess()
String snowflakeExpected =
" Relational\n" +
" (\n" +
" type = TDS[(Id, Integer, INT, \"\"), (Legal Name, String, \"\", \"\"), (Est Date, StrictDate, \"\", \"\"), (Mnc, Boolean, \"\", \"\"), (Employee Count, Integer, \"\", \"\"), (Last Update, DateTime, \"\", \"\")]\n" +
" type = TDS[(Id, Integer, INT, \"\"), (Legal Name, String, VARCHAR(65536), \"\"), (Est Date, StrictDate, DATE, \"\"), (Mnc, Boolean, BIT, \"\"), (Employee Count, Integer, BIGINT, \"\"), (Last Update, DateTime, TIMESTAMP, \"\")]\n" +
" resultColumns = [(\"Id\", INT), (\"Legal Name\", \"\"), (\"Est Date\", \"\"), (\"Mnc\", \"\"), (\"Employee Count\", \"\"), (\"Last Update\", \"\")]\n" +
" sql = select \"root\".ID as \"Id\", to_varchar(get_path(\"root\".FIRM_DETAILS, 'legalName')) as \"Legal Name\", to_date(get_path(\"root\".FIRM_DETAILS, 'dates.estDate')) as \"Est Date\", to_boolean(get_path(\"root\".FIRM_DETAILS, 'mnc')) as \"Mnc\", to_number(get_path(\"root\".FIRM_DETAILS, 'employeeCount')) as \"Employee Count\", to_timestamp(get_path(\"root\".FIRM_DETAILS, '[\"dates\"][\"last Update\"]')) as \"Last Update\" from FIRM_SCHEMA.FIRM_TABLE as \"root\"\n" +
" connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" +
" )\n";
String TDSType = " type = TDS[(Id, Integer, INT, \"\"), (Legal Name, String, \"\", \"\"), (Est Date, StrictDate, \"\", \"\"), (Mnc, Boolean, \"\", \"\"), (Employee Count, Integer, \"\", \"\"), (Last Update, DateTime, \"\", \"\")]\n";
String TDSType = " type = TDS[(Id, Integer, INT, \"\"), (Legal Name, String, VARCHAR(65536), \"\"), (Est Date, StrictDate, DATE, \"\"), (Mnc, Boolean, BIT, \"\"), (Employee Count, Integer, BIGINT, \"\"), (Last Update, DateTime, TIMESTAMP, \"\")]\n";
Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan);
}

Expand Down
Loading

0 comments on commit db9e8ea

Please sign in to comment.