Fix MemSQL semi-structured access for Integer types (#2600)

finos · Feb 11, 2024 · 9cf10e0 · 9cf10e0
1 parent d619fcc
commit 9cf10e0
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 39 deletions.
diff --git a/...rc/main/resources/core_relational_memsql/relational/sqlQueryToString/memSQLExtension.pure b/...rc/main/resources/core_relational_memsql/relational/sqlQueryToString/memSQLExtension.pure
@@ -157,7 +157,7 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::memsq
   if ($returnType == 'DATE', | 'date(json_extract_string(' + $fullPath + '))', |
   if ($returnType == 'BOOLEAN', | 'json_extract_json(' + $fullPath + ')', |
   if ($returnType == 'FLOAT', | 'json_extract_double(' + $fullPath + ')', |
-  if ($returnType == 'INTEGER', | 'json_extract_double(' + $fullPath + ')', |
+  if ($returnType == 'INTEGER', | 'json_extract_bigint(' + $fullPath + ')', |
   'json_extract_json(' + $fullPath + ')'))))));
 }
 
@@ -251,8 +251,9 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::memsq
 
   // MemSQL does not allow for fluent casting of date values, must explicitly cast the full operand after parsing as json string
   if($s.returnType == StrictDate, | 'date(' + $fullOperand + ')', |
+  if($s.returnType == Integer, | $fullOperand + ' !:> bigint', |
   if($s.returnType->isNotEmpty() && $s.returnType->toOne()->_subTypeOf(Date), | 'timestamp(' + $fullOperand + ')', |
-  $fullOperand));
+  $fullOperand)));
 }
 
 function <<access.private>> meta::relational::functions::sqlQueryToString::memsql::castOperatorForMemSQLSemiStructuredData(type:Type[0..1]): String[1]
@@ -261,9 +262,10 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::memsq
    // Dates treated as strings and explicitly casted after
    if($type == StrictDate, | '::$', |
    if($type->isNotEmpty() && $type->toOne()->_subTypeOf(Date), | '::$', |
+   if($type->isNotEmpty() && $type->toOne()->_subTypeOf(Integer), | '::', |
    if($type->isNotEmpty() && $type->toOne()->_subTypeOf(Number), | '::%', |
    if($type->isNotEmpty() && $type->toOne()->instanceOf(Enumeration), | '::$', |
-   '::')))));
+   '::'))))));
 }
 
 function <<access.private>> meta::relational::functions::sqlQueryToString::memsql::processUpsertSQLQueryForMemSQL(upsertQuery:UpsertSQLQuery[1], sgc:SqlGenerationContext[1]):String[1]

diff --git a/...an/execution/stores/relational/test/semiStructured/TestExtractFromSemiStructuredJoin.java b/...an/execution/stores/relational/test/semiStructured/TestExtractFromSemiStructuredJoin.java
@@ -44,7 +44,7 @@ public void testJoinOnSemiStructuredProperty()
                 "(\n" +
                 "  type = TDS[(First Name, String, VARCHAR(100), \"\"), (Last Name, String, VARCHAR(100), \"\"), (Firm/Legal Name, String, \"\", \"\")]\n" +
                 "  resultColumns = [(\"First Name\", VARCHAR(100)), (\"Last Name\", VARCHAR(100)), (\"Firm/Legal Name\", \"\")]\n" +
-                "  sql = select `root`.FIRSTNAME as `First Name`, `root`.LASTNAME as `Last Name`, `firm_table_0`.FIRM_DETAILS::$legalName as `Firm/Legal Name` from PERSON_SCHEMA.PERSON_TABLE as `root` left outer join FIRM_SCHEMA.FIRM_TABLE as `firm_table_0` on (json_extract_double(`root`.FIRM, 'ID') = json_extract_double(`firm_table_0`.FIRM_DETAILS, 'ID'))\n" +
+                "  sql = select `root`.FIRSTNAME as `First Name`, `root`.LASTNAME as `Last Name`, `firm_table_0`.FIRM_DETAILS::$legalName as `Firm/Legal Name` from PERSON_SCHEMA.PERSON_TABLE as `root` left outer join FIRM_SCHEMA.FIRM_TABLE as `firm_table_0` on (json_extract_bigint(`root`.FIRM, 'ID') = json_extract_bigint(`firm_table_0`.FIRM_DETAILS, 'ID'))\n" +
                 "  connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                 ")\n";
         Assert.assertEquals(memSQLExpected, memSQLPlan);

diff --git a/.../execution/stores/relational/test/semiStructured/TestExtractFromSemiStructuredSimple.java b/.../execution/stores/relational/test/semiStructured/TestExtractFromSemiStructuredSimple.java
@@ -101,17 +101,17 @@ public void testAllDataTypesAccess()
         String memSQLExpected =
                 "Relational\n" +
                 "(\n" +
-                "  type = TDS[(Id, Integer, INT, \"\"), (Legal Name, String, \"\", \"\"), (Est Date, StrictDate, \"\", \"\"), (Mnc, Boolean, \"\", \"\"), (Employee Count, Integer, \"\", \"\"), (Last Update, DateTime, \"\", \"\")]\n" +
-                "  resultColumns = [(\"Id\", INT), (\"Legal Name\", \"\"), (\"Est Date\", \"\"), (\"Mnc\", \"\"), (\"Employee Count\", \"\"), (\"Last Update\", \"\")]\n" +
-                "  sql = select `root`.ID as `Id`, json_extract_string(`root`.FIRM_DETAILS, 'legalName') as `Legal Name`, date(json_extract_string(`root`.FIRM_DETAILS, 'dates', 'estDate')) as `Est Date`, json_extract_json(`root`.FIRM_DETAILS, 'mnc') as `Mnc`, json_extract_double(`root`.FIRM_DETAILS, 'employeeCount') as `Employee Count`, timestamp(json_extract_string(`root`.FIRM_DETAILS, 'dates', 'last Update')) as `Last Update` from FIRM_SCHEMA.FIRM_TABLE as `root`\n" +
+                "  type = TDS[(Id, Integer, INT, \"\"), (Legal Name, String, \"\", \"\"), (Est Date, StrictDate, \"\", \"\"), (Mnc, Boolean, \"\", \"\"), (Employee Count, Integer, \"\", \"\"), (Revenue, Float, \"\", \"\"), (Last Update, DateTime, \"\", \"\")]\n" +
+                "  resultColumns = [(\"Id\", INT), (\"Legal Name\", \"\"), (\"Est Date\", \"\"), (\"Mnc\", \"\"), (\"Employee Count\", \"\"), (\"Revenue\", \"\"), (\"Last Update\", \"\")]\n" +
+                "  sql = select `root`.ID as `Id`, json_extract_string(`root`.FIRM_DETAILS, 'legalName') as `Legal Name`, date(json_extract_string(`root`.FIRM_DETAILS, 'dates', 'estDate')) as `Est Date`, json_extract_json(`root`.FIRM_DETAILS, 'mnc') as `Mnc`, json_extract_bigint(`root`.FIRM_DETAILS, 'employeeCount') as `Employee Count`, json_extract_double(`root`.FIRM_DETAILS, 'revenue') as `Revenue`, timestamp(json_extract_string(`root`.FIRM_DETAILS, 'dates', 'last Update')) as `Last Update` from FIRM_SCHEMA.FIRM_TABLE as `root`\n" +
                 "  connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                 ")\n";
         Assert.assertEquals(memSQLExpected, memSQLPlan);
 
         String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime);
-        Assert.assertEquals("1,Firm X,2010-03-04,true,4,2022-01-16 01:00:00.0\n" +
-                "2,Firm A,2012-11-13,false,1,2022-02-14 03:00:00.0\n" +
-                "3,Firm B,2017-07-07,true,2,2022-09-01 06:00:00.0\n", h2Result.replace("\r\n", "\n"));
+        Assert.assertEquals("1,Firm X,2010-03-04,true,4,5.0,2022-01-16 01:00:00.0\n" +
+                "2,Firm A,2012-11-13,false,1,2000.5,2022-02-14 03:00:00.0\n" +
+                "3,Firm B,2017-07-07,true,2,0.1,2022-09-01 06:00:00.0\n", h2Result.replace("\r\n", "\n"));
     }
 
     public String modelResourcePath()

diff --git a/...execution/stores/relational/test/semiStructured/TestSemiStructuredInheritanceMapping.java b/...execution/stores/relational/test/semiStructured/TestSemiStructuredInheritanceMapping.java
@@ -62,7 +62,7 @@ public void testSemiStructuredPropertyAccessAtSubClass()
                 "(\n" +
                 "  type = TDS[(First Name, String, VARCHAR(100), \"\"), (Firm Address 0 Line No, Integer, \"\", \"\")]\n" +
                 "  resultColumns = [(\"First Name\", VARCHAR(100)), (\"Firm Address 0 Line No\", \"\")]\n" +
-                "  sql = select `root`.FIRSTNAME as `First Name`, `root`.FIRM_DETAILS::address::`lines`::`0`::%lineno as `Firm Address 0 Line No` from PERSON_SCHEMA.PERSON_TABLE as `root`\n" +
+                "  sql = select `root`.FIRSTNAME as `First Name`, `root`.FIRM_DETAILS::address::`lines`::`0`::lineno !:> bigint as `Firm Address 0 Line No` from PERSON_SCHEMA.PERSON_TABLE as `root`\n" +
                 "  connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                 ")\n";
         Assert.assertEquals(memSQLExpected, memSQLPlan);
@@ -88,7 +88,7 @@ public void testSemiStructuredPropertyAccessAtSubClassNested()
                 "(\n" +
                 "  type = TDS[(First Name, String, VARCHAR(100), \"\"), (Firm Address 0 Line No, Integer, \"\", \"\"), (Firm Address Street, String, \"\", \"\"), (Firm Address City, String, \"\", \"\"), (Firm Address State, String, \"\", \"\")]\n" +
                 "  resultColumns = [(\"First Name\", VARCHAR(100)), (\"Firm Address 0 Line No\", \"\"), (\"Firm Address Street\", \"\"), (\"Firm Address City\", \"\"), (\"Firm Address State\", \"\")]\n" +
-                "  sql = select `root`.FIRSTNAME as `First Name`, `root`.FIRM_DETAILS::address::`lines`::`0`::%lineno as `Firm Address 0 Line No`, `root`.FIRM_DETAILS::address::`lines`::`0`::$street as `Firm Address Street`, `root`.FIRM_DETAILS::address::`lines`::`1`::$city as `Firm Address City`, `root`.FIRM_DETAILS::address::`lines`::`2`::$state as `Firm Address State` from PERSON_SCHEMA.PERSON_TABLE as `root`\n" +
+                "  sql = select `root`.FIRSTNAME as `First Name`, `root`.FIRM_DETAILS::address::`lines`::`0`::lineno !:> bigint as `Firm Address 0 Line No`, `root`.FIRM_DETAILS::address::`lines`::`0`::$street as `Firm Address Street`, `root`.FIRM_DETAILS::address::`lines`::`1`::$city as `Firm Address City`, `root`.FIRM_DETAILS::address::`lines`::`2`::$state as `Firm Address State` from PERSON_SCHEMA.PERSON_TABLE as `root`\n" +
                 "  connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                 ")\n";
         Assert.assertEquals(memSQLExpected, memSQLPlan);
@@ -114,7 +114,7 @@ public void testSemiStructuredPropertyAccessAtSubClassNestedUsingProjectWithFunc
                 "(\n" +
                 "  type = TDS[(First Name, String, VARCHAR(100), \"\"), (Firm Address 0 Line No, Integer, \"\", \"\"), (Firm Address Street, String, \"\", \"\"), (Firm Address City, String, \"\", \"\"), (Firm Address State, String, \"\", \"\")]\n" +
                 "  resultColumns = [(\"First Name\", VARCHAR(100)), (\"Firm Address 0 Line No\", \"\"), (\"Firm Address Street\", \"\"), (\"Firm Address City\", \"\"), (\"Firm Address State\", \"\")]\n" +
-                "  sql = select `root`.FIRSTNAME as `First Name`, `root`.FIRM_DETAILS::address::`lines`::`0`::%lineno as `Firm Address 0 Line No`, `root`.FIRM_DETAILS::address::`lines`::`0`::$street as `Firm Address Street`, `root`.FIRM_DETAILS::address::`lines`::`1`::$city as `Firm Address City`, `root`.FIRM_DETAILS::address::`lines`::`2`::$state as `Firm Address State` from PERSON_SCHEMA.PERSON_TABLE as `root`\n" +
+                "  sql = select `root`.FIRSTNAME as `First Name`, `root`.FIRM_DETAILS::address::`lines`::`0`::lineno !:> bigint as `Firm Address 0 Line No`, `root`.FIRM_DETAILS::address::`lines`::`0`::$street as `Firm Address Street`, `root`.FIRM_DETAILS::address::`lines`::`1`::$city as `Firm Address City`, `root`.FIRM_DETAILS::address::`lines`::`2`::$state as `Firm Address State` from PERSON_SCHEMA.PERSON_TABLE as `root`\n" +
                 "  connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                 ")\n";
         Assert.assertEquals(memSQLExpected, memSQLPlan);

diff --git a/...gine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredMatching.java b/...gine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredMatching.java
@@ -63,7 +63,7 @@ public void testSemiStructuredMatchWithMultipleProject()
                         "(\n" +
                         "  type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n" +
                         "  resultColumns = [(\"Customer Address\", \"\"), (\"Order Price\", \"\")]\n" +
-                        "  sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else null end as `Customer Address`, case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountPaid else null end as `Order Price` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" +
+                        "  sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else null end as `Customer Address`, case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::amountToBePaid !:> bigint when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::amountPaid !:> bigint else null end as `Order Price` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" +
                         "  connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                         ")\n";
         Assert.assertEquals(memSQLExpected, memSQLPlan);
@@ -91,7 +91,7 @@ public void testSemiStructuredMatchWithComplexFilter()
                         "(\n" +
                         "  type = TDS[(Customer Address, String, \"\", \"\")]\n" +
                         "  resultColumns = [(\"Customer Address\", \"\")]\n" +
-                        "  sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else null end as `Customer Address` from ORDER_SCHEMA.ORDER_TABLE as `root` where case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountPaid else null end < 200\n" +
+                        "  sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else null end as `Customer Address` from ORDER_SCHEMA.ORDER_TABLE as `root` where case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::amountToBePaid !:> bigint when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::amountPaid !:> bigint else null end < 200\n" +
                         "  connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                         ")\n";
         Assert.assertEquals(memSQLExpected, memSQLPlan);
@@ -136,7 +136,7 @@ public void testSemiStructuredMatchWithVariableAccess()
                         "    (\n" +
                         "      type = TDS[(Max Amount Flag, Boolean, \"\", \"\")]\n" +
                         "      resultColumns = [(\"Max Amount Flag\", \"\")]\n" +
-                        "      sql = select case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then case when `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid < ${maxAmount} then 'true' else 'false' end when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when `root`.CUSTOMER::transactionDetails::payment::%amountPaid < ${maxAmount} then 'true' else 'false' end else null end as `Max Amount Flag` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" +
+                        "      sql = select case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then case when `root`.CUSTOMER::transactionDetails::payment::amountToBePaid !:> bigint < ${maxAmount} then 'true' else 'false' end when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when `root`.CUSTOMER::transactionDetails::payment::amountPaid !:> bigint < ${maxAmount} then 'true' else 'false' end else null end as `Max Amount Flag` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" +
                         "      connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                         "    )\n" +
                         "  )\n" +
@@ -166,7 +166,7 @@ public void testSemiStructuredMatchMultilevel()
                         "(\n" +
                         "  type = TDS[(Amount, Integer, \"\", \"\")]\n" +
                         "  resultColumns = [(\"Amount\", \"\")]\n" +
-                        "  sql = select case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('WalletPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%walletTransactionAmount when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%cardTransactionAmount when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountPaid else null end when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid else null end as `Amount` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" +
+                        "  sql = select case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('WalletPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::walletTransactionAmount !:> bigint when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::cardTransactionAmount !:> bigint when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::amountPaid !:> bigint else null end when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::amountToBePaid !:> bigint else null end as `Amount` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" +
                         "  connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                         ")\n";
         Assert.assertEquals(memSQLExpected, memSQLPlan);
@@ -194,7 +194,7 @@ public void testSemiStructuredMatchWithMultipleProjectUsingCol()
                         "(\n" +
                         "  type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n" +
                         "  resultColumns = [(\"Customer Address\", \"\"), (\"Order Price\", \"\")]\n" +
-                        "  sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else null end as `Customer Address`, case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountPaid else null end as `Order Price` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" +
+                        "  sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else null end as `Customer Address`, case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::amountToBePaid !:> bigint when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::amountPaid !:> bigint else null end as `Order Price` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" +
                         "  connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" +
                         ")\n";
         Assert.assertEquals(memSQLExpected, memSQLPlan);