diff --git a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/test/pureClient/stores/Test_Relational_UsingPureClientTestSuite.java b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/test/pureClient/stores/Test_Relational_UsingPureClientTestSuite.java index 0700509ea2d..52fddc10fa0 100644 --- a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/test/pureClient/stores/Test_Relational_UsingPureClientTestSuite.java +++ b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/test/pureClient/stores/Test_Relational_UsingPureClientTestSuite.java @@ -84,6 +84,7 @@ public static Test suite() throws Exception suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStoreUnion::inMemoryAndRelational", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStore::ordered", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::functions::pureToSqlQuery::calendarAggregations", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::platform", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); return suite; }, diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/nodes/helpers/platform/DefaultExecutionNodeContext.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/nodes/helpers/platform/DefaultExecutionNodeContext.java index 73da77de274..b7461e9072d 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/nodes/helpers/platform/DefaultExecutionNodeContext.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/nodes/helpers/platform/DefaultExecutionNodeContext.java @@ -131,6 +131,10 @@ else if (result instanceof StreamingObjectResult) { return (T) ((StreamingObjectResult) result).getObjectStream(); } + else if (rawType.equals(List.class)) + { + return this.getResultAsParameterizedType(type, result.realizeInMemory()); + } } throw new IllegalArgumentException("Unable to convert " + result.getClass().getSimpleName() + " to " + TypeUtils.toString(type)); } diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/StreamingResult.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/StreamingResult.java index 62a55912e5e..5a1bf374293 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/StreamingResult.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/StreamingResult.java @@ -60,4 +60,15 @@ public String flush(Serializer serializer) throw new RuntimeException(e); } } + + public static long getRealizeRowLimit() + { + return Long.getLong( + "org.finos.legend.engine.realizedResultRowLimit", + Long.getLong( + "org.finos.legend.engine.realizedRelationalResultRowLimit", + 1_000L + ) + ); + } } diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/TDSResult.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/TDSResult.java index 76723f8fcea..0bc387b7256 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/TDSResult.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/TDSResult.java @@ -100,17 +100,6 @@ public Result realizeInMemory() } } - private static long getRealizeRowLimit() - { - return Long.getLong( - "org.finos.legend.engine.realizedResultRowLimit", - Long.getLong( - "org.finos.legend.engine.realizedRelationalResultRowLimit", - 1_000L - ) - ); - } - @Override public void cancel() { diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/object/StreamingObjectResult.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/object/StreamingObjectResult.java index 4cf14bf9b2d..1a08f214980 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/object/StreamingObjectResult.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/result/object/StreamingObjectResult.java @@ -14,6 +14,7 @@ package org.finos.legend.engine.plan.execution.result.object; +import org.eclipse.collections.api.factory.Lists; import org.finos.legend.engine.plan.dependencies.store.shared.IResult; import org.finos.legend.engine.plan.execution.result.ConstantResult; import org.finos.legend.engine.plan.execution.result.Result; @@ -23,6 +24,7 @@ import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat; import org.finos.legend.engine.plan.execution.result.serialization.Serializer; +import java.util.List; import java.util.stream.Stream; public class StreamingObjectResult extends StreamingResult @@ -91,4 +93,22 @@ public Serializer getSerializer(SerializationFormat format) throw new RuntimeException(format.toString() + " format not currently supported with StreamingObjectResult"); } } + + @Override + public Result realizeInMemory() + { + List res = Lists.mutable.empty(); + this.getObjectStream().forEach(x -> + { + if (res.size() > StreamingResult.getRealizeRowLimit()) + { + throw new RuntimeException("Too many rows returned. Realization of relational results currently supports results with up to " + getRealizeRowLimit() + " rows."); + } + else + { + res.add(x); + } + }); + return new ConstantResult(res); + } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/result/RealizedRelationalResult.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/result/RealizedRelationalResult.java index fe0ac2d20e5..e2a33977b24 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/result/RealizedRelationalResult.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/result/RealizedRelationalResult.java @@ -38,8 +38,6 @@ public class RealizedRelationalResult extends StreamingResult public List columns; public List> resultSetRows; public List> transformedRows; - - private static final int DEFAULT_ROW_LIMIT = 1000; public static final String ROW_LIMIT_PROPERTY_NAME = "org.finos.legend.engine.realizedRelationalResultRowLimit"; public RealizedRelationalResult(RelationalResult relationalResult) throws SQLException @@ -52,7 +50,7 @@ public RealizedRelationalResult(RelationalResult relationalResult) throws SQLExc this.transformedRows = Lists.mutable.empty(); this.resultSetRows = Lists.mutable.empty(); ResultSet resultSet = relationalResult.resultSet; - int SUPPORTED_RESULT_ROWS = getRowLimit(); + long SUPPORTED_RESULT_ROWS = getRealizeRowLimit(); int rowCount = 0; try { @@ -84,11 +82,6 @@ public RealizedRelationalResult(RelationalResult relationalResult) throws SQLExc } } - public int getRowLimit() - { - return Integer.getInteger(ROW_LIMIT_PROPERTY_NAME, DEFAULT_ROW_LIMIT); - } - private RealizedRelationalResult() { super(Lists.mutable.empty()); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/pom.xml index 94ddfbc3407..76980a7e571 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/pom.xml @@ -421,6 +421,21 @@ legend-engine-language-pure-compiler test + + org.finos.legend.engine + legend-engine-pure-functions-planExecution-pure + test + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-planExecution + test + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-shared-functions-planExecution + test + diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/platformOperations/testPlatformOperationsOnRelational.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/platformOperations/testPlatformOperationsOnRelational.pure new file mode 100644 index 00000000000..f3480e6dc82 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/platformOperations/testPlatformOperationsOnRelational.pure @@ -0,0 +1,83 @@ + +import meta::pure::profiles::*; +import meta::pure::mapping::*; +import meta::pure::profiles::*; + + +function <> meta::relational::tests::platform::operations::testIsNotEmptyForRelational_returnsTrue():Boolean[1] +{ + //CreateAndFill + meta::relational::tests::platform::operations::createTablesAndFillDbUS(); + + let finalQuery = + {| + + let employees = meta::relational::tests::platform::operations::Person.all() + ->filter( x | $x.region =='NYC')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS()); + + $employees->isNotEmpty(); + }; + + let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions()); + assertEquals('true', $result); +} + +function <> meta::relational::tests::platform::operations::testIsNotEmptyForRelational_returnsFalse():Boolean[1] +{ + //CreateAndFill + meta::relational::tests::platform::operations::createTablesAndFillDbUS(); + + + let finalQuery = + {| + + let employees = meta::relational::tests::platform::operations::Person.all() + ->filter( x | $x.region =='Chennai')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS()); + + $employees->isNotEmpty(); + }; + + let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions()); + assertEquals('false', $result); + +} + +function <> meta::relational::tests::platform::operations::testIsEmptyForRelational_returnsFalse():Boolean[1] +{ + //CreateAndFill + meta::relational::tests::platform::operations::createTablesAndFillDbUS(); + + + let finalQuery = + {| + + let employees = meta::relational::tests::platform::operations::Person.all() + ->filter( x | $x.region =='NYC')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS()); + + $employees->isEmpty(); + }; + + let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions()); + assertEquals('false', $result); + +} + +function <> meta::relational::tests::platform::operations::testIsEmptyForRelational_returnsTrue():Boolean[1] +{ + //CreateAndFill + meta::relational::tests::platform::operations::createTablesAndFillDbUS(); + + + let finalQuery = + {| + + let employees = meta::relational::tests::platform::operations::Person.all() + ->filter( x | $x.region =='Chennai')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS()); + + $employees->isEmpty(); + }; + + let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions()); + assertEquals('true', $result); + +} \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/platformOperations/testSetup.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/platformOperations/testSetup.pure new file mode 100644 index 00000000000..6836469e298 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/platformOperations/testSetup.pure @@ -0,0 +1,84 @@ +Class meta::relational::tests::platform::operations::Person +{ + firstName: String[1]; + lastName: String[*]; + isDataEng:Boolean[1]; + region:String[1]; +} + +function meta::relational::tests::platform::operations::createTablesAndFillDbUS():Boolean[1] +{ + let connection = meta::relational::tests::platform::operations::runtime::testRuntimeUS().connectionByElement(meta::relational::tests::platform::operations::database::EmployeeUSDB)->cast(@meta::external::store::relational::runtime::TestDatabaseConnection); + meta::relational::metamodel::execute::executeInDb('Drop table if exists Person;', $connection); + meta::relational::metamodel::execute::executeInDb('Create Table Person(PKEY INT PRIMARY KEY,FIRST_NAME VARCHAR(200),LAST_NAME VARCHAR(200),IS_FULLTIME BOOLEAN,REGION VARCHAR(200));', $connection); + meta::relational::metamodel::execute::executeInDb('insert into Person (PKEY, FIRST_NAME, LAST_NAME,IS_FULLTIME,REGION) values (1,\'John\', \'Smith\',True, \'NYC\');', $connection); + meta::relational::metamodel::execute::executeInDb('insert into Person (PKEY, FIRST_NAME, LAST_NAME,IS_FULLTIME,REGION) values (2,\'Jane\', \'Doe\',True, \'Chicago\');', $connection); + meta::relational::metamodel::execute::executeInDb('insert into Person (PKEY, FIRST_NAME, LAST_NAME,IS_FULLTIME,REGION) values (3,\'Christopher\', \'Nolan\',True, \'NYC\');', $connection); + true; +} + +###Relational +Database meta::relational::tests::platform::operations::database::EmployeeUSDB +( + Table Person ( + PKEY INT PRIMARY KEY, + FIRST_NAME VARCHAR(200) , + LAST_NAME VARCHAR(200) , + IS_FULLTIME BIT, + REGION VARCHAR(200) + ) +) + + +###Mapping +import meta::relational::tests::*; +import meta::external::store::relational::tests::*; +import meta::relational::tests::model::simple::*; + +Mapping meta::relational::tests::platform::operations::mapping::EmployeeUSMapping +( + meta::relational::tests::platform::operations::Person: Relational{ + ~primaryKey + ( + [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.PKEY + ) + ~mainTable [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person + firstName:[meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.FIRST_NAME, + lastName: [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.LAST_NAME, + isDataEng: [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.IS_FULLTIME, + region: [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.REGION + } +) + + +###Pure +import meta::relational::metamodel::*; +import meta::json::*; +import meta::json::tests::*; +import meta::relational::tests::*; +import meta::external::store::relational::tests::*; +import meta::pure::profiles::*; +import meta::relational::metamodel::execute::*; +import meta::core::runtime::*; +import meta::external::store::relational::runtime::*; +import meta::relational::runtime::*; + + +function meta::relational::tests::platform::operations::runtime::testRuntimeUS():Runtime[1] +{ + ^Runtime(connectionStores = meta::relational::tests::platform::operations::testDatabaseConnectionPerson(meta::relational::tests::platform::operations::database::EmployeeUSDB,'GMT')) +} + +function meta::relational::tests::platform::operations::testDatabaseConnectionPerson(db:Database[1], timeZone:String[0..1]) : ConnectionStore[1] +{ + ^ConnectionStore( + connection = ^meta::external::store::relational::runtime::TestDatabaseConnection( + type = DatabaseType.H2, + timeZone = if($timeZone->isEmpty(), |'GMT', |$timeZone) + ), + element = $db); +} + + + +