Review realization row limit

legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/test/pureClient/stores/Test_Relational_UsingPureClientTestSuite.java

@@ -84,6 +84,7 @@ public static Test suite() throws Exception
                     suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStoreUnion::inMemoryAndRelational", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport));
                     suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStore::ordered", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport));
                     suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::functions::pureToSqlQuery::calendarAggregations", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport));
+                    suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::platform", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport));
 
                     return suite;
                 },

legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/nodes/helpers/platform/DefaultExecutionNodeContext.java

@@ -131,6 +131,10 @@ else if (result instanceof StreamingObjectResult)
             {
                 return (T) ((StreamingObjectResult) result).getObjectStream();
             }
+            else if (rawType.equals(List.class))
+            {
+                return this.getResultAsParameterizedType(type, result.realizeInMemory());
+            }
         }
         throw new IllegalArgumentException("Unable to convert " + result.getClass().getSimpleName() + " to " + TypeUtils.toString(type));
     }

legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/StreamingResult.java

@@ -60,4 +60,15 @@ public String flush(Serializer serializer)
             throw new RuntimeException(e);
         }
     }
+
+    public static long getRealizeRowLimit()
+    {
+            return Long.getLong(
+                            "org.finos.legend.engine.realizedResultRowLimit",
+                            Long.getLong(
+                                            "org.finos.legend.engine.realizedRelationalResultRowLimit",
+                                            1_000L
+                                            )
+                            );
+    }
 }

legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/TDSResult.java

@@ -100,17 +100,6 @@ public Result realizeInMemory()
         }
     }
 
-    private static long getRealizeRowLimit()
-    {
-        return Long.getLong(
-                "org.finos.legend.engine.realizedResultRowLimit",
-                Long.getLong(
-                        "org.finos.legend.engine.realizedRelationalResultRowLimit",
-                        1_000L
-                )
-        );
-    }
-
     @Override
     public void cancel()
     {

legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/object/StreamingObjectResult.java

@@ -14,6 +14,7 @@
 
 package org.finos.legend.engine.plan.execution.result.object;
 
+import org.eclipse.collections.api.factory.Lists;
 import org.finos.legend.engine.plan.dependencies.store.shared.IResult;
 import org.finos.legend.engine.plan.execution.result.ConstantResult;
 import org.finos.legend.engine.plan.execution.result.Result;
@@ -23,6 +24,7 @@
 import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat;
 import org.finos.legend.engine.plan.execution.result.serialization.Serializer;
 
+import java.util.List;
 import java.util.stream.Stream;
 
 public class StreamingObjectResult<T> extends StreamingResult
@@ -91,4 +93,22 @@ public Serializer getSerializer(SerializationFormat format)
             throw new RuntimeException(format.toString() + " format not currently supported with StreamingObjectResult");
         }
     }
+
+    @Override
+    public Result realizeInMemory()
+    {
+        List<T> res = Lists.mutable.empty();
+        this.getObjectStream().forEach(x ->
+        {
+            if (res.size() > StreamingResult.getRealizeRowLimit())
+            {
+                throw new RuntimeException("Too many rows returned. Realization of relational results currently supports results with up to " + getRealizeRowLimit() + " rows.");
+            }
+            else
+            {
+                res.add(x);
+            }
+        });
+        return new ConstantResult(res);
+    }
 }

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/result/RealizedRelationalResult.java

@@ -38,8 +38,6 @@ public class RealizedRelationalResult extends StreamingResult
     public List<SQLResultColumn> columns;
     public List<List<Object>> resultSetRows;
     public List<List<Object>> transformedRows;
-
-    private static final int DEFAULT_ROW_LIMIT = 1000;
     public static final String ROW_LIMIT_PROPERTY_NAME = "org.finos.legend.engine.realizedRelationalResultRowLimit";
 
     public RealizedRelationalResult(RelationalResult relationalResult) throws SQLException
@@ -52,7 +50,7 @@ public RealizedRelationalResult(RelationalResult relationalResult) throws SQLExc
         this.transformedRows = Lists.mutable.empty();
         this.resultSetRows = Lists.mutable.empty();
         ResultSet resultSet = relationalResult.resultSet;
-        int SUPPORTED_RESULT_ROWS = getRowLimit();
+        long SUPPORTED_RESULT_ROWS = getRealizeRowLimit();
         int rowCount = 0;
         try
         {
@@ -84,11 +82,6 @@ public RealizedRelationalResult(RelationalResult relationalResult) throws SQLExc
         }
     }
 
-    public int getRowLimit()
-    {
-        return Integer.getInteger(ROW_LIMIT_PROPERTY_NAME, DEFAULT_ROW_LIMIT);
-    }
-
     private RealizedRelationalResult()
     {
         super(Lists.mutable.empty());

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/pom.xml

@@ -421,6 +421,21 @@
             <artifactId>legend-engine-language-pure-compiler</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-functions-planExecution-pure</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-runtime-java-extension-compiled-functions-planExecution</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-runtime-java-extension-shared-functions-planExecution</artifactId>
+            <scope>test</scope>
+        </dependency>
 
         <!-- TEST -->
     </dependencies>

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/platformOperations/testPlatformOperationsOnRelational.pure

@@ -0,0 +1,83 @@
+
+import meta::pure::profiles::*;
+import meta::pure::mapping::*;
+import meta::pure::profiles::*;
+
+
+function <<test.Test,test.AlloyOnly>> meta::relational::tests::platform::operations::testIsNotEmptyForRelational_returnsTrue():Boolean[1]
+{
+  //CreateAndFill
+  meta::relational::tests::platform::operations::createTablesAndFillDbUS();
+
+  let finalQuery = 
+    {|
+
+      let employees = meta::relational::tests::platform::operations::Person.all()
+                                      ->filter( x | $x.region =='NYC')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS());
+
+      $employees->isNotEmpty();
+    };
+
+  let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions());
+  assertEquals('true', $result);
+}
+
+function <<test.Test,test.AlloyOnly>> meta::relational::tests::platform::operations::testIsNotEmptyForRelational_returnsFalse():Boolean[1]
+{
+  //CreateAndFill
+  meta::relational::tests::platform::operations::createTablesAndFillDbUS();
+
+
+  let finalQuery = 
+    {|
+
+      let employees = meta::relational::tests::platform::operations::Person.all()
+                                      ->filter( x | $x.region =='Chennai')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS());
+
+      $employees->isNotEmpty();
+    };
+
+  let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions());
+  assertEquals('false', $result);
+
+}
+
+function <<test.Test,test.AlloyOnly>> meta::relational::tests::platform::operations::testIsEmptyForRelational_returnsFalse():Boolean[1]
+{
+  //CreateAndFill
+  meta::relational::tests::platform::operations::createTablesAndFillDbUS();
+
+
+  let finalQuery = 
+    {|
+
+      let employees = meta::relational::tests::platform::operations::Person.all()
+                                      ->filter( x | $x.region =='NYC')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS());
+
+      $employees->isEmpty();
+    };
+
+  let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions());
+  assertEquals('false', $result);
+
+}
+
+function <<test.Test,test.AlloyOnly>> meta::relational::tests::platform::operations::testIsEmptyForRelational_returnsTrue():Boolean[1]
+{
+  //CreateAndFill
+  meta::relational::tests::platform::operations::createTablesAndFillDbUS();
+
+
+  let finalQuery = 
+    {|
+
+      let employees = meta::relational::tests::platform::operations::Person.all()
+                                      ->filter( x | $x.region =='Chennai')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS());
+
+      $employees->isEmpty();
+    };
+
+  let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions());
+  assertEquals('true', $result);
+
+}

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/platformOperations/testSetup.pure

@@ -0,0 +1,84 @@
+Class meta::relational::tests::platform::operations::Person
+{
+  firstName: String[1];
+  lastName: String[*];
+  isDataEng:Boolean[1];
+  region:String[1];
+}
+
+function meta::relational::tests::platform::operations::createTablesAndFillDbUS():Boolean[1]
+{
+  let connection = meta::relational::tests::platform::operations::runtime::testRuntimeUS().connectionByElement(meta::relational::tests::platform::operations::database::EmployeeUSDB)->cast(@meta::external::store::relational::runtime::TestDatabaseConnection);
+   meta::relational::metamodel::execute::executeInDb('Drop table if exists Person;', $connection);
+   meta::relational::metamodel::execute::executeInDb('Create Table Person(PKEY  INT PRIMARY KEY,FIRST_NAME VARCHAR(200),LAST_NAME VARCHAR(200),IS_FULLTIME BOOLEAN,REGION VARCHAR(200));', $connection);
+   meta::relational::metamodel::execute::executeInDb('insert into Person (PKEY, FIRST_NAME, LAST_NAME,IS_FULLTIME,REGION) values (1,\'John\', \'Smith\',True, \'NYC\');', $connection);
+   meta::relational::metamodel::execute::executeInDb('insert into Person (PKEY, FIRST_NAME, LAST_NAME,IS_FULLTIME,REGION) values (2,\'Jane\', \'Doe\',True, \'Chicago\');', $connection);
+   meta::relational::metamodel::execute::executeInDb('insert into Person (PKEY, FIRST_NAME, LAST_NAME,IS_FULLTIME,REGION) values (3,\'Christopher\', \'Nolan\',True, \'NYC\');', $connection);
+  true; 
+}
+
+###Relational
+Database meta::relational::tests::platform::operations::database::EmployeeUSDB
+(
+    Table Person (
+      PKEY  INT PRIMARY KEY,
+      FIRST_NAME VARCHAR(200) ,
+      LAST_NAME VARCHAR(200) ,
+      IS_FULLTIME BIT,
+      REGION VARCHAR(200)
+      )
+)
+
+
+###Mapping
+import meta::relational::tests::*;
+import meta::external::store::relational::tests::*;
+import meta::relational::tests::model::simple::*;
+
+Mapping meta::relational::tests::platform::operations::mapping::EmployeeUSMapping
+(
+ meta::relational::tests::platform::operations::Person: Relational{
+   ~primaryKey
+    (
+      [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.PKEY
+    )
+    ~mainTable [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person
+    firstName:[meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.FIRST_NAME,
+    lastName: [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.LAST_NAME,
+    isDataEng: [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.IS_FULLTIME,
+    region: [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.REGION
+  }
+)
+
+
+###Pure
+import meta::relational::metamodel::*;
+import meta::json::*;
+import meta::json::tests::*;
+import meta::relational::tests::*;
+import meta::external::store::relational::tests::*;
+import meta::pure::profiles::*;
+import meta::relational::metamodel::execute::*;
+import meta::core::runtime::*;
+import meta::external::store::relational::runtime::*;
+import meta::relational::runtime::*;
+
+
+function meta::relational::tests::platform::operations::runtime::testRuntimeUS():Runtime[1]
+{
+  ^Runtime(connectionStores = meta::relational::tests::platform::operations::testDatabaseConnectionPerson(meta::relational::tests::platform::operations::database::EmployeeUSDB,'GMT'))
+}
+
+function meta::relational::tests::platform::operations::testDatabaseConnectionPerson(db:Database[1], timeZone:String[0..1]) : ConnectionStore[1]
+{
+  ^ConnectionStore(
+    connection = ^meta::external::store::relational::runtime::TestDatabaseConnection(
+        type = DatabaseType.H2,
+        timeZone = if($timeZone->isEmpty(), |'GMT', |$timeZone)
+    ),
+    element = $db);
+}
+
+
+
+