From fdd3b4036f9882a2d230603d839176dcb2b653c1 Mon Sep 17 00:00:00 2001 From: Pierre De Belen Date: Fri, 26 Jul 2024 14:37:35 -0400 Subject: [PATCH] Add support for OLAP functions (#2984) --- .../pom.xml | 10 - .../toPureGraph/RelationTypeHelper.java | 3 +- .../ValueSpecificationBuilder.java | 14 +- .../toPureGraph/handlers/Handlers.java | 25 +- .../pure/serialization/toPureGrammar.pure | 11 +- .../relation/functions/extend.pure | 410 +++++++++++++++++- .../relation/functions/first.pure | 17 + .../relation/functions/lag.pure | 26 ++ .../relation/functions/last.pure | 17 + .../relation/functions/lead.pure | 26 ++ .../relation/functions/offset.pure | 17 + .../relation/functions/olap/frame.pure | 19 + .../relation/functions/olap/over.pure | 79 ++++ .../relation/functions/olap/range.pure | 22 + .../relation/functions/olap/rows.pure | 22 + .../relation/functions/rank.pure | 17 + .../compiled/RelationExtensionCompiled.java | 10 +- .../RelationNativeImplementation.java | 267 +++++++++--- .../relation/compiled/natives/Extend.java | 14 +- .../relation/compiled/natives/ExtendAgg.java | 50 +++ .../compiled/natives/ExtendAggArray.java | 37 ++ .../compiled/natives/ExtendArray.java | 4 +- .../compiled/natives/ExtendWindowAgg.java | 52 +++ .../natives/ExtendWindowAggArray.java | 40 ++ .../compiled/natives/ExtendWindowFunc.java | 70 +++ .../natives/ExtendWindowFuncArray.java | 37 ++ .../relation/compiled/natives/First.java | 27 ++ .../relation/compiled/natives/GroupBy.java | 15 +- .../compiled/natives/GroupByArray.java | 2 +- .../relation/compiled/natives/Offset.java | 28 ++ .../relation/compiled/natives/Project.java | 2 +- .../relation/compiled/natives/Sort.java | 12 +- .../natives/shared/NullRowContainer.java | 24 + .../compiled/natives/shared/RowContainer.java | 9 + .../natives/shared/TestTDSCompiled.java | 3 +- .../relation/compiled/natives/shared/Win.java | 46 ++ .../pure}/TestFunctionTester.java | 51 +-- .../RelationExtensionInterpreted.java | 12 + .../relation/interpreted/natives/Extend.java | 202 ++++++--- .../relation/interpreted/natives/First.java | 51 +++ .../relation/interpreted/natives/GroupBy.java | 116 +---- .../relation/interpreted/natives/Last.java | 52 +++ .../relation/interpreted/natives/Offset.java | 67 +++ .../relation/interpreted/natives/Sort.java | 9 +- .../natives/shared/AggregationShared.java | 150 +++++++ .../shared/TDSWithCursorCoreInstance.java | 5 + .../interpreted/pure/TestFunctionTester.java | 232 ++++++++++ .../external/relation/shared/ColumnValue.java | 84 ++++ .../external/relation/shared/TestTDS.java | 84 +++- .../finos/legend/engine/ide/PureIDELight.java | 28 +- .../legend-engine-pure-ide-light-pure/pom.xml | 11 - .../Test_JAVA_RelationFunction_PCT.java | 14 +- ...lational_DuckDB_RelationFunctions_PCT.java | 15 +- ...t_Relational_H2_RelationFunctions_PCT.java | 15 +- ...tional_Postgres_RelationFunctions_PCT.java | 16 +- ...ional_Snowflake_RelationFunctions_PCT.java | 15 +- .../RelationalCompilerExtension.java | 77 ++-- pom.xml | 2 +- 58 files changed, 2439 insertions(+), 353 deletions(-) create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/first.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lag.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/last.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lead.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/offset.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/frame.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/over.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/range.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/rows.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/rank.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendAgg.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendAggArray.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowAgg.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowAggArray.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowFunc.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowFuncArray.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/First.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Offset.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/NullRowContainer.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/Win.java rename legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/{ => compiled/pure}/TestFunctionTester.java (76%) create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/First.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Last.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Offset.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/AggregationShared.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/interpreted/pure/TestFunctionTester.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/ColumnValue.java diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml index 4d6f892d87f..2ebf7a8cbd5 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml @@ -126,21 +126,11 @@ org.finos.legend.engine legend-engine-pure-platform-dsl-store-java - - org.finos.legend.engine - legend-engine-pure-runtime-java-extension-compiled-functions-relation - runtime - org.finos.legend.pure legend-pure-runtime-java-engine-compiled - - org.finos.legend.engine - legend-engine-pure-runtime-java-extension-compiled-functions-unclassified - runtime - org.finos.legend.engine diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationTypeHelper.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationTypeHelper.java index 03627a6a5df..e98fdb65e18 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationTypeHelper.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationTypeHelper.java @@ -17,6 +17,7 @@ import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.engine.protocol.pure.v1.model.relationType.Column; import org.finos.legend.engine.protocol.pure.v1.model.relationType.RelationType; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.Multiplicity; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; import org.finos.legend.pure.m3.navigation.ProcessorSupport; import org.finos.legend.pure.m3.navigation._package._Package; @@ -42,6 +43,6 @@ public static RelationType convert(org.finos.legend.pure.m3.coreinstance.meta.pu public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType convert(RelationType src, ProcessorSupport processorSupport, SourceInformation sourceInformation) { - return _RelationType.build(ListIterate.collect(src.columns, c -> (CoreInstance) _Column.getColumnInstance(c.name, false, (GenericType) processorSupport.type_wrapGenericType(_Package.getByUserPath(c.type, processorSupport)), sourceInformation, processorSupport)).toList(), sourceInformation, processorSupport); + return _RelationType.build(ListIterate.collect(src.columns, c -> (CoreInstance) _Column.getColumnInstance(c.name, false, (GenericType) processorSupport.type_wrapGenericType(_Package.getByUserPath(c.type, processorSupport)), (Multiplicity) org.finos.legend.pure.m3.navigation.multiplicity.Multiplicity.newMultiplicity(0, 1, processorSupport), sourceInformation, processorSupport)).toList(), sourceInformation, processorSupport); } } diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/ValueSpecificationBuilder.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/ValueSpecificationBuilder.java index 85e15c6c73e..dbd3e600226 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/ValueSpecificationBuilder.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/ValueSpecificationBuilder.java @@ -304,7 +304,7 @@ private ValueSpecification proccessColSpecArray(ColSpecArray value) cols.collect(c -> { Column theCol = ((RelationType) c._genericType()._typeArguments().getLast()._rawType())._columns().getFirst(); - return _Column.getColumnInstance(theCol._name(), false, _Column.getColumnType(theCol), null, processorSupport); + return _Column.getColumnInstance(theCol._name(), false, _Column.getColumnType(theCol), _Column.getColumnMultiplicity(theCol), null, processorSupport); }), null, processorSupport @@ -351,7 +351,7 @@ private ValueSpecification proccessColSpec(ColSpec colSpec) ProcessorSupport processorSupport = context.pureModel.getExecutionSupport().getProcessorSupport(); if (colSpec.function1 == null) { - return wrapInstanceValue(buildColSpec(colSpec.name, colSpec.type == null ? null : context.pureModel.getGenericType(colSpec.type, colSpec.sourceInformation), context.pureModel, context.pureModel.getExecutionSupport().getProcessorSupport()), context.pureModel); + return wrapInstanceValue(buildColSpec(colSpec.name, colSpec.type == null ? null : context.pureModel.getGenericType(colSpec.type, colSpec.sourceInformation), (Multiplicity) org.finos.legend.pure.m3.navigation.multiplicity.Multiplicity.newMultiplicity(0, 1, processorSupport), context.pureModel, context.pureModel.getExecutionSupport().getProcessorSupport()), context.pureModel); } else if (colSpec.function2 == null) { @@ -365,7 +365,7 @@ else if (colSpec.function2 == null) new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) ._rawType( _RelationType.build( - Lists.mutable.with(_Column.getColumnInstance(colSpec.name, false, funcReturnType(funcVS, context.pureModel), null, processorSupport)), + Lists.mutable.with(_Column.getColumnInstance(colSpec.name, false, funcReturnType(funcVS, context.pureModel), funcReturnMul(funcVS, context.pureModel), null, processorSupport)), null, processorSupport ) @@ -405,7 +405,7 @@ else if (colSpec.function2 == null) new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) ._rawType( _RelationType.build( - Lists.mutable.with(_Column.getColumnInstance(colSpec.name, false, funcReturnType(func2VS, context.pureModel), null, processorSupport)), + Lists.mutable.with(_Column.getColumnInstance(colSpec.name, false, funcReturnType(func2VS, context.pureModel), funcReturnMul(func2VS, context.pureModel), null, processorSupport)), null, processorSupport ) @@ -712,9 +712,9 @@ public ValueSpecification processClassInstance(RootGraphFetchTree rootGraphFetch { org.finos.legend.pure.m3.coreinstance.meta.pure.graphFetch.GraphFetchTree tree = HelperValueSpecificationBuilder.buildGraphFetchTree(rootGraphFetchTree, this.context, null, openVariables, processingContext); GenericType genericType = new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) - ._rawType(this.context.pureModel.getType("meta::pure::graphFetch::RootGraphFetchTree")) - ._typeArguments(FastList.newListWith(new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) - ._rawType(context.resolveClass(rootGraphFetchTree._class, rootGraphFetchTree.sourceInformation)))); + ._rawType(this.context.pureModel.getType("meta::pure::graphFetch::RootGraphFetchTree")) + ._typeArguments(FastList.newListWith(new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) + ._rawType(context.resolveClass(rootGraphFetchTree._class, rootGraphFetchTree.sourceInformation)))); return new Root_meta_pure_metamodel_valuespecification_InstanceValue_Impl("", SourceInformationHelper.toM3SourceInformation(rootGraphFetchTree.sourceInformation), context.pureModel.getClass("meta::pure::metamodel::valuespecification::InstanceValue")) ._genericType(genericType) ._multiplicity(this.context.pureModel.getMultiplicity("one")) diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java index 40f54a488b5..2153d896782 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java @@ -257,7 +257,7 @@ private static TypeAndMultiplicity getTypeAndMultiplicity(MutableList relType = _RelationType.build( types.flatCollect(RelationTypeAccessor::_columns) - .collect(c -> _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), null, processorSupport)), + .collect(c -> _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), _Column.getColumnMultiplicity(c), null, processorSupport)), null, processorSupport ); @@ -297,7 +297,7 @@ public static TypeAndMultiplicity ExtendReturnInference(List Lists.mutable .withAll((RichIterable>) ((RelationType) ps.get(0)._genericType()._typeArguments().getFirst()._rawType())._columns()) .withAll((RichIterable>) ((RelationType) ps.get(1)._genericType()._typeArguments().getLast()._rawType())._columns()) - .collect(c -> (CoreInstance) _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), null, processorSupport)), + .collect(c -> (CoreInstance) _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), _Column.getColumnMultiplicity(c), null, processorSupport)), null, processorSupport ); @@ -354,7 +354,7 @@ public static TypeAndMultiplicity ExtendReturnInference(List return Lists.mutable.with( vs, wrapInstanceValue(buildColSpec(foundColumn, cc.pureModel, ps), cc.pureModel), - wrapInstanceValue(buildColSpec(secondCol.name, _Column.getColumnType(foundColumn), cc.pureModel, ps), cc.pureModel) + wrapInstanceValue(buildColSpec(secondCol.name, _Column.getColumnType(foundColumn), _Column.getColumnMultiplicity(foundColumn), cc.pureModel, ps), cc.pureModel) ); }; @@ -391,12 +391,12 @@ public static InstanceValue wrapInstanceValue(MutableList values, ._values(values); } - public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpec buildColSpec(String name, GenericType colType, PureModel pureModel, ProcessorSupport ps) + public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpec buildColSpec(String name, GenericType colType, Multiplicity multiplicity, PureModel pureModel, ProcessorSupport ps) { GenericType firstGenericType = new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) ._rawType( _RelationType.build( - Lists.mutable.with(_Column.getColumnInstance(name, false, colType, null, ps)), + Lists.mutable.with(_Column.getColumnInstance(name, false, colType, multiplicity, null, ps)), null, ps ) @@ -413,16 +413,16 @@ public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpec buildColSpec(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.Column col, PureModel pureModel, ProcessorSupport ps) { - return buildColSpec(col._name(), _Column.getColumnType(col), pureModel, ps); + return buildColSpec(col._name(), _Column.getColumnType(col), _Column.getColumnMultiplicity(col), pureModel, ps); } public static final ParametersInference LambdaColCollectionInference = (parameters, ov, cc, pc) -> { ValueSpecification firstProcessedParameter = parameters.get(0).accept(new ValueSpecificationBuilder(cc, ov, pc)); GenericType gt = firstProcessedParameter._genericType(); - if (parameters.get(1) instanceof ClassInstance) + if (parameters.get(1) instanceof ClassInstance) { - ((ColSpecArray)((ClassInstance) parameters.get(1)).value).colSpecs.forEach(col -> updateSimpleLambda(col.function1, gt, new org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity(1, 1))); + ((ColSpecArray) ((ClassInstance) parameters.get(1)).value).colSpecs.forEach(col -> updateSimpleLambda(col.function1, gt, new org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity(1, 1))); } else { @@ -1108,11 +1108,11 @@ private void registerTDS() { if (c._name().equals(firstCol._name())) { - return (CoreInstance) _Column.getColumnInstance(secondCol._name(), false, _Column.getColumnType(c), null, processorSupport); + return (CoreInstance) _Column.getColumnInstance(secondCol._name(), false, _Column.getColumnType(c), _Column.getColumnMultiplicity(c), null, processorSupport); } else { - return (CoreInstance) _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), null, processorSupport); + return (CoreInstance) _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), _Column.getColumnMultiplicity(c), null, processorSupport); } }).toList(), null, @@ -1897,6 +1897,11 @@ public static GenericType funcReturnType(ValueSpecification vs, PureModel pm) return funcType(vs._genericType(), pm)._returnType(); } + public static Multiplicity funcReturnMul(ValueSpecification vs, PureModel pm) + { + return funcType(vs._genericType(), pm)._returnMultiplicity(); + } + private GenericType funcReturnType(ValueSpecification vs) { return funcType(vs._genericType(), this.pureModel)._returnType(); diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure index 8d562764d87..ba6413389bf 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure @@ -256,7 +256,16 @@ function meta::pure::metamodel::serialization::grammar::printGenericType(generic { if ($genericType->instanceOf(GenericTypeOperation), |let go = $genericType->cast(@GenericTypeOperation); - $go.left->printGenericType()+if($go.type == GenericTypeOperationType.Union,|'+',|'-')+$go.right->printGenericType();, + if($go.left->isEmpty(),|'',|$go.left->toOne()->printGenericType()) + + if( + [ + pair(|$go.type == GenericTypeOperationType.Union,|'+'), + pair(|$go.type == GenericTypeOperationType.Difference,|'-'), + pair(|$go.type == GenericTypeOperationType.Subset,|'⊆') + ], + |'=' + ) + + $go.right->printGenericType();, |if ($genericType.rawType->isEmpty(), |if ($genericType.typeParameter->isEmpty(), |'', diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/extend.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/extend.pure index 20408134e6c..0a6a58500bd 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/extend.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/extend.pure @@ -16,9 +16,17 @@ import meta::pure::test::pct::*; import meta::pure::metamodel::relation::*; native function <> meta::pure::functions::relation::extend(r:Relation[1], f:FuncColSpec<{T[1]->Any[0..1]},Z>[1]):Relation[1]; - native function <> meta::pure::functions::relation::extend(r:Relation[1], fs:FuncColSpecArray<{T[1]->Any[*]},Z>[1]):Relation[1]; +native function <> meta::pure::functions::relation::extend(r:Relation[1], agg:AggColSpec<{T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1]; +native function <> meta::pure::functions::relation::extend(r:Relation[1], agg:AggColSpecArray<{T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1]; + +native function <> meta::pure::functions::relation::extend(r:Relation[1], window:_Window[1], f:FuncColSpec<{Relation[1],T[1]->Any[0..1]},R>[1]):Relation[1]; +native function <> meta::pure::functions::relation::extend(r:Relation[1], window:_Window[1], f:FuncColSpecArray<{Relation[1],T[1]->Any[*]},R>[1]):Relation[1]; + +native function <> meta::pure::functions::relation::extend(r:Relation[1], window:_Window[1], agg:AggColSpec<{Relation[1],T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1]; +native function <> meta::pure::functions::relation::extend(r:Relation[1], window:_Window[1], agg:AggColSpecArray<{Relation[1],T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1]; + function <> meta::pure::functions::relation::tests::extend::testSimpleExtendStrShared(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] { @@ -119,3 +127,403 @@ function <> meta::pure::functions::relation::tests::extend::testSimple ' 6,weq,7,weq_ext\n'+ '#', $res->toString()); } + + +function <> meta::pure::functions::relation::tests::extend::testOLAPAggNoWindow(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(~newCol:c|$c.id:y|$y->plus()) + }; + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol\n'+ + ' 1,2,A,55\n'+ + ' 2,1,B,55\n'+ + ' 3,3,C,55\n'+ + ' 4,4,D,55\n'+ + ' 5,2,E,55\n'+ + ' 6,1,F,55\n'+ + ' 7,3,G,55\n'+ + ' 8,1,H,55\n'+ + ' 9,5,I,55\n'+ + ' 10,0,J,55\n'+ + '#', $res->sort(~id->ascending())->toString()); +} + + +function <> meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowMultipleColums(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(~[newCol:c|$c.id:y|$y->plus(), other:c|$c.id:y|$y->plus() +1]) + }; + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol,other\n'+ + ' 1,2,A,55,56\n'+ + ' 2,1,B,55,56\n'+ + ' 3,3,C,55,56\n'+ + ' 4,4,D,55,56\n'+ + ' 5,2,E,55,56\n'+ + ' 6,1,F,55,56\n'+ + ' 7,3,G,55,56\n'+ + ' 8,1,H,55,56\n'+ + ' 9,5,I,55,56\n'+ + ' 10,0,J,55,56\n'+ + '#', $res->sort(~id->ascending())->toString()); +} + + +function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindow(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(over(~grp), ~newCol:{w,c|$c.id}:y|$y->plus()) + }; + + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol\n'+ + ' 10,0,J,10\n'+ + ' 2,1,B,16\n'+ + ' 6,1,F,16\n'+ + ' 8,1,H,16\n'+ + ' 1,2,A,6\n'+ + ' 5,2,E,6\n'+ + ' 3,3,C,10\n'+ + ' 7,3,G,10\n'+ + ' 4,4,D,4\n'+ + ' 9,5,I,9\n'+ + '#', $res->sort(~grp->ascending())->toString()); +} + +function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindowMultipleColumns(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(over(~grp), ~[newCol:{w,c|$c.id}:y|$y->plus(), other:{w,c|$c.id}:y|$y->plus()+1]) + }; + + let res = $f->eval($expr); + + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol,other\n'+ + ' 10,0,J,10,11\n'+ + ' 2,1,B,16,17\n'+ + ' 6,1,F,16,17\n'+ + ' 8,1,H,16,17\n'+ + ' 1,2,A,6,7\n'+ + ' 5,2,E,6,7\n'+ + ' 3,3,C,10,11\n'+ + ' 7,3,G,10,11\n'+ + ' 4,4,D,4,5\n'+ + ' 9,5,I,9,10\n'+ + '#', $res->sort(~grp->ascending())->toString()); +} + +function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindow(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(over(~grp, ~id->descending()), ~newCol:{w,c|$c.name}:y|$y->joinStrings('')) + }; + + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol\n'+ + ' 10,0,J,J\n'+ + ' 8,1,H,HFB\n'+ + ' 6,1,F,HFB\n'+ + ' 2,1,B,HFB\n'+ + ' 5,2,E,EA\n'+ + ' 1,2,A,EA\n'+ + ' 7,3,G,GC\n'+ + ' 3,3,C,GC\n'+ + ' 4,4,D,D\n'+ + ' 9,5,I,I\n'+ + '#', $res->sort(~grp->ascending())->toString()); +} + +function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowUsingLag(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(over(~grp, ~id->descending()), ~newCol:{w,c|$w->lag($c).name}:y|$y->joinStrings('')) + }; + + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol\n'+ + ' 10,0,J,\n'+ + ' 8,1,H,HF\n'+ + ' 6,1,F,HF\n'+ + ' 2,1,B,HF\n'+ + ' 5,2,E,E\n'+ + ' 1,2,A,E\n'+ + ' 7,3,G,G\n'+ + ' 3,3,C,G\n'+ + ' 4,4,D,\n'+ + ' 9,5,I,\n'+ + '#', $res->sort(~grp->ascending())->toString()); +} + +function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowMultipleColumns(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(over(~grp, ~id->descending()), ~[newCol:{w,c|$c.name}:y|$y->joinStrings(''),other:{w,x|$x.id}:y|$y->plus()]) + }; + + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol,other\n'+ + ' 10,0,J,J,10\n'+ + ' 8,1,H,HFB,16\n'+ + ' 6,1,F,HFB,16\n'+ + ' 2,1,B,HFB,16\n'+ + ' 5,2,E,EA,6\n'+ + ' 1,2,A,EA,6\n'+ + ' 7,3,G,GC,10\n'+ + ' 3,3,C,GC,10\n'+ + ' 4,4,D,D,4\n'+ + ' 9,5,I,I,9\n'+ + '#', $res->sort(~grp->ascending())->toString()); +} + + +function <> meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindow(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(over(~grp, ~id->descending()), ~newCol:{w,c|$w->lead($c).id}) + }; + + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol\n'+ + ' 10,0,J,null\n'+ + ' 8,1,H,6\n'+ + ' 6,1,F,2\n'+ + ' 2,1,B,null\n'+ + ' 5,2,E,1\n'+ + ' 1,2,A,null\n'+ + ' 7,3,G,3\n'+ + ' 3,3,C,null\n'+ + ' 4,4,D,null\n'+ + ' 9,5,I,null\n'+ + '#', $res->sort(~grp->ascending())->toString()); +} + +function <> meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindowMultipleColumns(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(over(~grp, ~id->descending()), ~[newCol:{w,c|$w->lead($c).id}, other:{w,c|$w->first().name}]) + }; + + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol,other\n'+ + ' 10,0,J,null,J\n'+ + ' 8,1,H,6,H\n'+ + ' 6,1,F,2,H\n'+ + ' 2,1,B,null,H\n'+ + ' 5,2,E,1,E\n'+ + ' 1,2,A,null,E\n'+ + ' 7,3,G,3,G\n'+ + ' 3,3,C,null,G\n'+ + ' 4,4,D,null,D\n'+ + ' 9,5,I,null,I\n'+ + '#', $res->sort(~grp->ascending())->toString()); +} + + +function <> meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowChainedWithSimple(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, name + 1, 2, A + 2, 1, B + 3, 3, C + 4, 4, D + 5, 2, E + 6, 1, F + 7, 3, G + 8, 1, H + 9, 5, I + 10, 0, J + #->extend(~newCol:c|$c.id:y|$y->plus()) + ->extend(~other:x|floor($x.newCol->toOne() / $x.id->toOne())) + }; + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,name,newCol,other\n'+ + ' 1,2,A,55,55\n'+ + ' 2,1,B,55,27\n'+ + ' 3,3,C,55,18\n'+ + ' 4,4,D,55,13\n'+ + ' 5,2,E,55,11\n'+ + ' 6,1,F,55,9\n'+ + ' 7,3,G,55,7\n'+ + ' 8,1,H,55,6\n'+ + ' 9,5,I,55,6\n'+ + ' 10,0,J,55,5\n'+ + '#', $res->sort(~id->ascending())->toString()); +} + +function <> meta::pure::functions::relation::tests::extend::testOLAPWithMultiplePartitionsAndOrderWindowMultipleColumns(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let expr = { + | #TDS + id, grp, grp2, name + 1, 2, 2, A + 2, 1, 1, B + 3, 3, 3, C + 4, 4, 4, D + 5, 2, 2, E + 6, 1, 2, F + 7, 3, 3, G + 8, 1, 1, H + 9, 5, 5, I + 10, 0, 0, J + #->extend(over(~[grp,grp2], ~id->descending()), ~[newCol:{w,c|$w->lead($c).id}, other:{w,c|$w->first().name}]) + }; + + + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' id,grp,grp2,name,newCol,other\n'+ + ' 10,0,0,J,null,J\n'+ + ' 8,1,1,H,2,H\n'+ + ' 2,1,1,B,null,H\n'+ + ' 6,1,2,F,null,F\n'+ + ' 5,2,2,E,1,E\n'+ + ' 1,2,2,A,null,E\n'+ + ' 7,3,3,G,3,G\n'+ + ' 3,3,3,C,null,G\n'+ + ' 4,4,4,D,null,D\n'+ + ' 9,5,5,I,null,I\n'+ + '#', $res->sort([~grp->ascending(), ~grp2->ascending()])->toString()); +} + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/first.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/first.pure new file mode 100644 index 00000000000..fdd94226ca9 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/first.pure @@ -0,0 +1,17 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::metamodel::relation::*; + +native function meta::pure::functions::relation::first(w:Relation[1]):T[1]; diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lag.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lag.pure new file mode 100644 index 00000000000..86b29ad754d --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lag.pure @@ -0,0 +1,26 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::metamodel::relation::*; +import meta::pure::functions::relation::*; + +function meta::pure::functions::relation::lag(w:Relation[1],r:T[1]):T[1] +{ + lag($w, $r, 1); +} + +function meta::pure::functions::relation::lag(w:Relation[1],r:T[1], offset:Integer[1]):T[1] +{ + offset($w, $r, -$offset); +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/last.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/last.pure new file mode 100644 index 00000000000..b08b6a436b3 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/last.pure @@ -0,0 +1,17 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::metamodel::relation::*; + +native function meta::pure::functions::relation::last(w:Relation[1]):T[1]; \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lead.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lead.pure new file mode 100644 index 00000000000..94aa75af546 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lead.pure @@ -0,0 +1,26 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::metamodel::relation::*; +import meta::pure::functions::relation::*; + +function meta::pure::functions::relation::lead(w:Relation[1],r:T[1]):T[1] +{ + lead($w, $r, 1) +} + +function meta::pure::functions::relation::lead(w:Relation[1],r:T[1], offset:Integer[1]):T[1] +{ + offset($w, $r, $offset); +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/offset.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/offset.pure new file mode 100644 index 00000000000..7caaa0fd751 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/offset.pure @@ -0,0 +1,17 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::metamodel::relation::*; + +native function meta::pure::functions::relation::offset(w:Relation[1], r:T[1], offset:Integer[1]):T[1]; diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/frame.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/frame.pure new file mode 100644 index 00000000000..1bff304f5ff --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/frame.pure @@ -0,0 +1,19 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +Class meta::pure::functions::relation::Frame +{ + offsetFrom : Integer[1]; + offsetTo : Integer[1]; +} diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/over.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/over.pure new file mode 100644 index 00000000000..e7ae76ae112 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/over.pure @@ -0,0 +1,79 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::metamodel::relation::*; + +Class meta::pure::functions::relation::_Window +{ + partition : String[*]; + sortInfo : SortInfo[*]; + frame : Frame[0..1]; +} + +function meta::pure::functions::relation::over(cols:String[*], sortInfo:SortInfo<(?:?)⊆T>[*], frame:Frame[0..1]):_Window[1] +{ + ^_Window + ( + partition=$cols, + sortInfo=$sortInfo, + frame = $frame + ); +} + +function meta::pure::functions::relation::over(cols:ColSpec<(?:?)⊆T>[1]):_Window[1] +{ + ^_Window + ( + partition=$cols.name + ); +} + +function meta::pure::functions::relation::over(cols:ColSpecArray<(?:?)⊆T>[1]):_Window[1] +{ + ^_Window + ( + partition=$cols.names + ); +} + + +function meta::pure::functions::relation::over(sortInfo:SortInfo<(?:?)⊆T>[*]):_Window[1] +{ + over([],$sortInfo,[]) +} + +function meta::pure::functions::relation::over(frame:Frame[1]):_Window[1] +{ + over([],[],$frame) +} + +function meta::pure::functions::relation::over(cols:ColSpec<(?:?)⊆T>[1], sortInfo:SortInfo<(?:?)⊆T>[*]):_Window[1] +{ + over($cols.name,$sortInfo,[]) +} + +function meta::pure::functions::relation::over(cols:ColSpecArray<(?:?)⊆T>[1], sortInfo:SortInfo<(?:?)⊆T>[*]):_Window[1] +{ + over($cols.names,$sortInfo,[]) +} + +function meta::pure::functions::relation::over(cols:ColSpec<(?:?)⊆T>[1], frame:Frame[1]):_Window[1] +{ + over($cols.name,[],$frame) +} + +function meta::pure::functions::relation::over(sortInfo:SortInfo<(?:?)⊆T>[*], frame:Frame[1]):_Window[1] +{ + over([],$sortInfo,$frame) +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/range.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/range.pure new file mode 100644 index 00000000000..d18f7978a85 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/range.pure @@ -0,0 +1,22 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +Class meta::pure::functions::relation::_Range extends Frame +{ +} + +function meta::pure::functions::relation::_range(offsetFrom:Integer[1], offsetTo:Integer[1]):_Range[1] +{ + ^_Range(offsetFrom = $offsetFrom, offsetTo = $offsetTo); +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/rows.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/rows.pure new file mode 100644 index 00000000000..3d37289edc2 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/rows.pure @@ -0,0 +1,22 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +Class meta::pure::functions::relation::Rows extends Frame +{ +} + +function meta::pure::functions::relation::rows(offsetFrom:Integer[1], offsetTo:Integer[1]):Rows[1] +{ + ^Rows(offsetFrom = $offsetFrom, offsetTo = $offsetTo); +} diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/rank.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/rank.pure new file mode 100644 index 00000000000..410e9acc63f --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/rank.pure @@ -0,0 +1,17 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::metamodel::relation::*; + +native function meta::pure::functions::relation::rank(w:Relation[1],r:T[1]):Integer[1]; \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java index 2ae6a9b6663..e06c1c6f474 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java @@ -51,6 +51,13 @@ public List getExtraNatives() new Join(), new Extend(), new ExtendArray(), + new ExtendAgg(), + new ExtendAggArray(), + new ExtendWindowAgg(), + new ExtendWindowAggArray(), + new ExtendWindowFunc(), + new ExtendWindowFuncArray(), + new First(), new Drop(), new Sort(), new Rename(), @@ -60,7 +67,8 @@ public List getExtraNatives() new Slice(), new Distinct(), new Select(), - new SelectArray() + new SelectArray(), + new Offset() ); } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java index e2efb5ea2ee..900b3465afd 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java @@ -18,6 +18,7 @@ import org.eclipse.collections.api.RichIterable; import org.eclipse.collections.api.block.function.Function2; import org.eclipse.collections.api.block.function.Function3; +import org.eclipse.collections.api.block.procedure.Procedure2; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Sets; import org.eclipse.collections.api.list.ListIterable; @@ -34,9 +35,12 @@ import org.finos.legend.pure.m4.coreinstance.CoreInstance; import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; import org.finos.legend.pure.runtime.java.compiled.generation.processors.support.CompiledSupport; +import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.NullRowContainer; import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.RowContainer; import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.TDSContainer; import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.TestTDSCompiled; +import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.Win; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.ColumnValue; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortDirection; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortInfo; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; @@ -45,6 +49,7 @@ public class RelationNativeImplementation { + public static TestTDSCompiled getTDS(Object value) { return value instanceof TDSContainer ? @@ -120,7 +125,7 @@ public static Relation select(Relation r, Col public static Relation concatenate(Relation rel1, Relation rel2, ExecutionSupport es) { ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); - return new TDSContainer((TestTDSCompiled) RelationNativeImplementation.getTDS(rel1).concatenate((TestTDSCompiled) RelationNativeImplementation.getTDS(rel2)), ps); + return new TDSContainer((TestTDSCompiled) RelationNativeImplementation.getTDS(rel1).concatenate(RelationNativeImplementation.getTDS(rel2)), ps); } public static Relation filter(Relation rel, Function2 pureFunction, ExecutionSupport es) @@ -138,58 +143,164 @@ public static Relation filter(Relation rel, Functi return new TDSContainer((TestTDSCompiled) tds.drop(list), ps); } - public static class ColFuncSpecTrans + public static T offset(Relation w, T r, long offset, ExecutionSupport es) + { + int actualOffset = ((RowContainer) r).getRow() + (int) offset; + if (actualOffset < 0 || actualOffset >= ((TDSContainer) w).tds.getRowCount()) + { + return (T) new NullRowContainer(); + } + return (T) new RowContainer(RelationNativeImplementation.getTDS(w), actualOffset); + } + + public static Object first(Relation rel, ExecutionSupport es) + { + return new RowContainer(RelationNativeImplementation.getTDS(rel), 0); + } + + public abstract static class ColFuncSpecTrans { public String newColName; - public Function2 func; public String columnType; - public ColFuncSpecTrans(String newColName, Function2 func, String columnType) + public ColFuncSpecTrans(String newColName, String columnType) { this.newColName = newColName; - this.func = func; this.columnType = columnType; } + + public abstract Object eval(Object win, Object row, ExecutionSupport es); + } + + public static class ColFuncSpecTrans1 extends ColFuncSpecTrans + { + public Function2 func; + + public ColFuncSpecTrans1(String newColName, Function2 func, String columnType) + { + super(newColName, columnType); + this.func = func; + } + + @Override + public Object eval(Object win, Object row, ExecutionSupport es) + { + return func.value(row, es); + } } - public static Relation extend(Relation rel, MutableList colFuncSpecTrans, ExecutionSupport es) + public static class ColFuncSpecTrans2 extends ColFuncSpecTrans + { + public Function3 func; + + public ColFuncSpecTrans2(String newColName, Function3 func, String columnType) + { + super(newColName, columnType); + this.func = func; + } + + @Override + public Object eval(Object win, Object row, ExecutionSupport es) + { + return func.value(win, row, es); + } + } + + public static Relation extend(Relation rel, MutableList colFuncSpecTrans, ExecutionSupport es) + { + ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); + TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel); + return new TDSContainer((TestTDSCompiled) colFuncSpecTrans.injectInto((TestTDS) tds, (accTDS, colFuncSpec) -> accTDS.addColumn(performExtend(tds.wrapFullTDS(), colFuncSpec, es))), ps); + } + + public static Relation extendAgg(Relation rel, MutableList aggColSpecTrans, ExecutionSupport es) { ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel); - TestTDSCompiled t = colFuncSpecTrans.injectInto(tds, (a, b) -> performExtend(b, es, a, ps)); - return new TDSContainer(t, ps); + return new TDSContainer((TestTDSCompiled) aggregateTDS(tds.wrapFullTDS(), aggColSpecTrans, false, es).injectInto((TestTDS) tds, TestTDS::addColumn), ps); + } + + public static Relation extendWinFunc(Relation rel, Win window, MutableList colFunc, ExecutionSupport es) + { + ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); + TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel); + + Pair>> sortRes = tds.sort(window.getPartition().collect(part -> new SortInfo(part, SortDirection.ASC)).toList()); + final Pair>> sortedPartitions = TestTDS.sortPartitions(window.getSorts(), sortRes); + + return new TDSContainer((TestTDSCompiled) colFunc.injectInto(sortedPartitions.getOne(), (a, b) -> a.addColumn(performExtend(sortedPartitions, b, es))), ps); } - private static TestTDSCompiled performExtend(ColFuncSpecTrans colFuncSpecTrans, ExecutionSupport es, TestTDSCompiled tds, ProcessorSupport ps) + public static Relation extendWinAgg(Relation rel, Win window, MutableList aggColSpecTrans, ExecutionSupport es) { + ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); + TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel); + + Pair>> sortRes = tds.sort(window.getPartition().collect(part -> new SortInfo(part, SortDirection.ASC)).toList()); + final Pair>> sortedPartitions = TestTDS.sortPartitions(window.getSorts(), sortRes); + + return new TDSContainer((TestTDSCompiled) aggregateTDS(sortedPartitions, aggColSpecTrans, false, es).injectInto(sortedPartitions.getOne(), TestTDS::addColumn), ps); + } + + + private static ColumnValue performExtend(Pair>> tds, ColFuncSpecTrans colFuncSpecTrans, ExecutionSupport es) + { + long size = tds.getOne().getRowCount(); + boolean[] nulls = new boolean[(int) size]; switch (colFuncSpecTrans.columnType) { case "String": MutableList res = Lists.mutable.empty(); - for (int i = 0; i < tds.getRowCount(); i++) - { - res.add((String) colFuncSpecTrans.func.value(new RowContainer(tds, i), es)); - } - return (TestTDSCompiled) tds.addColumn(colFuncSpecTrans.newColName, DataType.STRING, res.toArray(new String[0])); + extracted(tds, colFuncSpecTrans, es, (i, val) -> res.add((String) val)); + return new ColumnValue(colFuncSpecTrans.newColName, DataType.STRING, res.toArray(new String[0])); case "Integer": - int[] resultInt = new int[(int) tds.getRowCount()]; - for (int i = 0; i < tds.getRowCount(); i++) - { - resultInt[i] = (int) (long) colFuncSpecTrans.func.value(new RowContainer(tds, i), es); - } - return (TestTDSCompiled) tds.addColumn(colFuncSpecTrans.newColName, DataType.INT, resultInt); + int[] resultInt = new int[(int) size]; + extracted(tds, colFuncSpecTrans, es, (i, val) -> processWithNull(i, val, nulls, () -> resultInt[i] = (int) (long) val)); + return new ColumnValue(colFuncSpecTrans.newColName, DataType.INT, resultInt, nulls); case "Double": case "Float": - double[] resultDouble = new double[(int) tds.getRowCount()]; - for (int i = 0; i < tds.getRowCount(); i++) - { - resultDouble[i] = (double) colFuncSpecTrans.func.value(new RowContainer(tds, i), es); - } - return (TestTDSCompiled) tds.addColumn(colFuncSpecTrans.newColName, DataType.DOUBLE, resultDouble); + double[] resultDouble = new double[(int) size]; + extracted(tds, colFuncSpecTrans, es, (i, val) -> processWithNull(i, val, nulls, () -> resultDouble[i] = (double) val)); + return new ColumnValue(colFuncSpecTrans.newColName, DataType.DOUBLE, resultDouble, nulls); } throw new RuntimeException(colFuncSpecTrans.columnType + " not supported yet"); } + private static void processWithNull(Integer j, Object val, boolean[] nulls, Proc p) + { + { + if (val == null) + { + nulls[j] = true; + } + else + { + p.invoke(); + } + } + } + + private interface Proc + { + void invoke(); + } + + private static void extracted(Pair>> tds, ColFuncSpecTrans colFuncSpecTrans, ExecutionSupport es, Procedure2 func) + { + int size = tds.getTwo().size(); + int k = 0; + for (int j = 0; j < size; j++) + { + Pair r = tds.getTwo().get(j); + TDSContainer winTDS = new TDSContainer((TestTDSCompiled) tds.getOne().slice(r.getOne(), r.getTwo()), ((CompiledExecutionSupport) es).getProcessorSupport()); + for (int i = 0; i < r.getTwo() - r.getOne(); i++) + { + func.value(k++, colFuncSpecTrans.eval(winTDS, new RowContainer((TestTDSCompiled) tds.getOne(), i), es)); + } + } + } + + public static Relation join(Relation rel1, Relation rel2, Enum joinKind, Function3 pureFunction, ExecutionSupport es) { ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); @@ -222,90 +333,146 @@ public static Relation sort(Relation rel, RichIter return new TDSContainer((TestTDSCompiled) tds1.sort(collect.collect(c -> new SortInfo(c.getTwo(), SortDirection.valueOf(c.getOne()._name()))).toList()).getOne(), ps); } - public static class AggColSpecTrans + public abstract static class AggColSpecTrans { public String newColName; - public Function2 map; public Function2 reduce; public String reduceType; - public AggColSpecTrans(String newColName, Function2 map, Function2 reduce, String reduceType) + public AggColSpecTrans(String newColName, Function2 reduce, String reduceType) { this.newColName = newColName; - this.map = map; this.reduce = reduce; this.reduceType = reduceType; } + + public abstract Object eval(Object win, Object row, ExecutionSupport es); } - public static Relation groupBy(Relation rel, ColSpec cols, MutableList aggColSpecTrans, ExecutionSupport es) + public static class AggColSpecTrans1 extends AggColSpecTrans + { + public Function2 map; + + public AggColSpecTrans1(String newColName, Function2 map, Function2 reduce, String reduceType) + { + super(newColName, reduce, reduceType); + this.map = map; + } + + @Override + public Object eval(Object win, Object row, ExecutionSupport es) + { + return map.value(row, es); + } + } + + public static class AggColSpecTrans2 extends AggColSpecTrans + { + public Function3 map; + + public AggColSpecTrans2(String newColName, Function3 map, Function2 reduce, String reduceType) + { + super(newColName, reduce, reduceType); + this.map = map; + } + + @Override + public Object eval(Object win, Object row, ExecutionSupport es) + { + return map.value(win, row, es); + } + } + + public static Relation groupBy(Relation rel, ColSpec cols, MutableList aggColSpecTrans, ExecutionSupport es) { return groupBy(rel, Lists.mutable.with(cols._name()), aggColSpecTrans, es); } - public static Relation groupBy(Relation rel, ColSpecArray cols, MutableList aggColSpecTransAll, ExecutionSupport es) + public static Relation groupBy(Relation rel, ColSpecArray cols, MutableList aggColSpecTransAll, ExecutionSupport es) { return groupBy(rel, Lists.mutable.withAll(cols._names()), aggColSpecTransAll, es); } - private static Relation groupBy(Relation rel, MutableList cols, MutableList aggColSpecTransAll, ExecutionSupport es) + private static Relation groupBy(Relation rel, MutableList cols, MutableList aggColSpecTransAll, ExecutionSupport es) { ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel); Pair>> sortRes = tds.sort(cols.collect(name -> new SortInfo(name, SortDirection.ASC)).toList()); - int size = sortRes.getTwo().size(); - MutableSet columnsToRemove = tds.getColumnNames().clone().toSet(); columnsToRemove.removeAll(cols.toSet()); + TestTDS distinctTDS = sortRes.getOne()._distinct(sortRes.getTwo()).removeColumns(columnsToRemove); - TestTDSCompiled finalTDS = (TestTDSCompiled) sortRes.getOne()._distinct(sortRes.getTwo()).removeColumns(columnsToRemove); + return new TDSContainer((TestTDSCompiled) aggregateTDS(sortRes, aggColSpecTransAll, true, es).injectInto(distinctTDS, TestTDS::addColumn), ps); + } + private static MutableList aggregateTDS(Pair>> sortRes, MutableList aggColSpecTransAll, boolean compress, ExecutionSupport es) + { + int size = compress ? sortRes.getTwo().size() : (int) sortRes.getOne().getRowCount(); + MutableList columnValues = Lists.mutable.empty(); for (AggColSpecTrans aggColSpecTrans : aggColSpecTransAll) { - switch ((String) aggColSpecTrans.reduceType) + switch (aggColSpecTrans.reduceType) { case "String": String[] finalRes = new String[size]; - performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sortRes, (o, j) -> finalRes[j] = (String) o); - finalTDS.addColumn(aggColSpecTrans.newColName, DataType.STRING, finalRes); + performMapReduce(aggColSpecTrans, aggColSpecTrans.reduce, es, sortRes, (o, j) -> finalRes[j] = (String) o, compress); + columnValues.add(new ColumnValue(aggColSpecTrans.newColName, DataType.STRING, finalRes)); break; case "Integer": int[] finalResInt = new int[size]; - performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sortRes, (o, j) -> finalResInt[j] = (int) (long) o); - finalTDS.addColumn(aggColSpecTrans.newColName, DataType.INT, finalResInt); + performMapReduce(aggColSpecTrans, aggColSpecTrans.reduce, es, sortRes, (o, j) -> finalResInt[j] = (int) (long) o, compress); + columnValues.add(new ColumnValue(aggColSpecTrans.newColName, DataType.INT, finalResInt)); break; case "Double": case "Float": case "Number": double[] finalResDouble = new double[size]; - performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sortRes, (o, j) -> finalResDouble[j] = (double) o); - finalTDS.addColumn(aggColSpecTrans.newColName, DataType.FLOAT, finalResDouble); + performMapReduce(aggColSpecTrans, aggColSpecTrans.reduce, es, sortRes, (o, j) -> finalResDouble[j] = (double) o, compress); + columnValues.add(new ColumnValue(aggColSpecTrans.newColName, DataType.FLOAT, finalResDouble)); break; default: throw new RuntimeException(aggColSpecTrans.reduceType + " is not supported yet!"); } } - - return new TDSContainer(finalTDS, ps); + return columnValues; } - private static void performMapReduce(Function2 map, Function2 reduce, ExecutionSupport es, int size, Pair>> sortRes, Function2 val) + private static void performMapReduce(AggColSpecTrans map, Function2 reduce, ExecutionSupport es, Pair>> sortRes, Function2 val, boolean compress) { + int cursor = 0; + int size = sortRes.getTwo().size(); for (int j = 0; j < size; j++) { Pair r = sortRes.getTwo().get(j); MutableList subList = org.eclipse.collections.impl.factory.Lists.mutable.empty(); - for (int i = r.getOne(); i < r.getTwo(); i++) + TDSContainer winTDS = new TDSContainer((TestTDSCompiled) sortRes.getOne().slice(r.getOne(), r.getTwo()), ((CompiledExecutionSupport) es).getProcessorSupport()); + for (int i = 0; i < r.getTwo() - r.getOne(); i++) + { + Object res = map.eval(winTDS, new RowContainer(winTDS.tds, i), es); + if (res != null) + { + subList.add(res); + } + } + // Write result + Object result = reduce.value(subList, es); + if (compress) { - subList.add(map.value(new RowContainer((TestTDSCompiled) sortRes.getOne(), i), es)); + val.apply(result, j); + } + else + { + for (int i = r.getOne(); i < r.getTwo(); i++) + { + val.apply(result, cursor++); + } } - val.apply(reduce.value(subList, es), j); } } - public static Relation project(RichIterable objects, RichIterable colFuncs, ExecutionSupport es) + public static Relation project(RichIterable objects, RichIterable colFuncs, ExecutionSupport es) { ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Extend.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Extend.java index 065826a205e..321ba92fe3c 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Extend.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Extend.java @@ -40,23 +40,25 @@ static StringBuilder buildCode(ListIterable transformedParams, Function< StringBuilder result = new StringBuilder("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.extend("); result.append(transformedParams.get(0) + ", "); result.append(collection.valueOf(transformedParams.get(1))); - buildCollectFuncSpec(result); + buildCollectFuncSpec(result, false); result.append(", es)"); return result; } - static void buildCollectFuncSpec(StringBuilder result) + static void buildCollectFuncSpec(StringBuilder result, boolean twoArgs) { + String className = "ColFuncSpecTrans" + (twoArgs ? "2" : "1"); + String functionType = "org.eclipse.collections.api.block.function.Function" + (twoArgs ? "3" : "2"); result.append(".collect("); - result.append("new DefendedFunction, org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.ColFuncSpecTrans>()\n" + + result.append("new DefendedFunction, org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation." + className + ">()\n" + "{\n" + " @Override\n" + - " public org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.ColFuncSpecTrans valueOf(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.FuncColSpec c)\n" + + " public org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation." + className + " valueOf(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.FuncColSpec c)\n" + " {\n"); - result.append("return new org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.ColFuncSpecTrans("); + result.append("return new org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation." + className + "("); result.append("c._name(),"); - result.append("(Function2)PureCompiledLambda.getPureFunction(c._function(),es),"); + result.append("(" + functionType + ")PureCompiledLambda.getPureFunction(c._function(),es),"); result.append(" ((org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType)c._function()._classifierGenericType()._typeArguments().toList().get(0)._rawType())._returnType()._rawType()._name()\n"); result.append(");\n"); diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendAgg.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendAgg.java new file mode 100644 index 00000000000..a0e4b737375 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendAgg.java @@ -0,0 +1,50 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; + +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.list.ListIterable; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.compiled.generation.ProcessorContext; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; + +import static org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.GroupBy.processAggColSpec; + +public class ExtendAgg extends AbstractNative implements Native +{ + public ExtendAgg() + { + super("extend_Relation_1__AggColSpec_1__Relation_1_"); + } + + @Override + public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) + { + StringBuilder result = buildCode(transformedParams, s -> "Lists.mutable.with(" + transformedParams.get(1) + ")"); + return result.toString(); + } + + static StringBuilder buildCode(ListIterable transformedParams, Function collection) + { + StringBuilder result = new StringBuilder("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.extendAgg("); + result.append(transformedParams.get(0) + ", "); + result.append(collection.valueOf(transformedParams.get(1))); + processAggColSpec(result, false); + result.append(", es)"); + return result; + } +} + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendAggArray.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendAggArray.java new file mode 100644 index 00000000000..2d5e602a22e --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendAggArray.java @@ -0,0 +1,37 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; + +import org.eclipse.collections.api.list.ListIterable; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.compiled.generation.ProcessorContext; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; + +public class ExtendAggArray extends AbstractNative implements Native +{ + public ExtendAggArray() + { + super("extend_Relation_1__AggColSpecArray_1__Relation_1_"); + } + + @Override + public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) + { + StringBuilder result = ExtendAgg.buildCode(transformedParams, s -> "Lists.mutable.withAll(" + transformedParams.get(1) + "._aggSpecs())"); + return result.toString(); + } +} + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendArray.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendArray.java index d8e56c0907d..d407e1b4e9c 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendArray.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendArray.java @@ -20,8 +20,6 @@ import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; -import static org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.Extend.buildCode; - public class ExtendArray extends AbstractNative implements Native { public ExtendArray() @@ -32,7 +30,7 @@ public ExtendArray() @Override public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) { - StringBuilder result = buildCode(transformedParams, s -> "Lists.mutable.withAll(" + transformedParams.get(1) + "._funcSpecs())"); + StringBuilder result = Extend.buildCode(transformedParams, s -> "Lists.mutable.withAll(" + transformedParams.get(1) + "._funcSpecs())"); return result.toString(); } } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowAgg.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowAgg.java new file mode 100644 index 00000000000..5e17c234aea --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowAgg.java @@ -0,0 +1,52 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; + +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.list.ListIterable; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.compiled.generation.ProcessorContext; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; + +import static org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.GroupBy.processAggColSpec; + +public class ExtendWindowAgg extends AbstractNative implements Native +{ + public ExtendWindowAgg() + { + super("extend_Relation_1___Window_1__AggColSpec_1__Relation_1_"); + } + + @Override + public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) + { + StringBuilder result = buildCode(transformedParams, s -> "Lists.mutable.with(" + transformedParams.get(2) + ")"); + return result.toString(); + } + + static StringBuilder buildCode(ListIterable transformedParams, Function collection) + { + StringBuilder result = new StringBuilder("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.extendWinAgg("); + result.append(transformedParams.get(0) + ", "); + ExtendWindowFunc.processWindow(result, transformedParams.get(1)); + result.append(","); + result.append(collection.valueOf(transformedParams.get(2))); + processAggColSpec(result, true); + result.append(", es)"); + return result; + } +} + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowAggArray.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowAggArray.java new file mode 100644 index 00000000000..8ebe2fd9cfd --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowAggArray.java @@ -0,0 +1,40 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; + +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.list.ListIterable; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.compiled.generation.ProcessorContext; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; + +import static org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.GroupBy.processAggColSpec; + +public class ExtendWindowAggArray extends AbstractNative implements Native +{ + public ExtendWindowAggArray() + { + super("extend_Relation_1___Window_1__AggColSpecArray_1__Relation_1_"); + } + + @Override + public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) + { + StringBuilder result = ExtendWindowAgg.buildCode(transformedParams, s -> "Lists.mutable.withAll(" + transformedParams.get(2) + "._aggSpecs())"); + return result.toString(); + } +} + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowFunc.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowFunc.java new file mode 100644 index 00000000000..281a5780620 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowFunc.java @@ -0,0 +1,70 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; + +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.list.ListIterable; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.compiled.generation.ProcessorContext; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; + +import static org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.Extend.buildCollectFuncSpec; +import static org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.GroupBy.processAggColSpec; + +public class ExtendWindowFunc extends AbstractNative implements Native +{ + public ExtendWindowFunc() + { + super("extend_Relation_1___Window_1__FuncColSpec_1__Relation_1_"); + } + + @Override + public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) + { + StringBuilder result = buildCode(transformedParams, s -> "Lists.mutable.with(" + transformedParams.get(2) + ")"); + return result.toString(); + } + + static StringBuilder buildCode(ListIterable transformedParams, Function collection) + { + StringBuilder result = new StringBuilder("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.extendWinFunc("); + result.append(transformedParams.get(0) + ", "); + processWindow(result, transformedParams.get(1)); + result.append(","); + result.append(collection.valueOf(transformedParams.get(2))); + buildCollectFuncSpec(result, true); + result.append(", es)"); + return result; + } + + public static void processWindow(StringBuilder result, String param) + { + result.append("Lists.mutable.with(" + param + ").collect(new DefendedFunction, org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.Win>()\n" + + "{\n" + + " @Override\n" + + " public org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.Win valueOf(Root_meta_pure_functions_relation__Window rootMetaPureFunctionsRelation__window)\n" + + " {\n" + + " return "); + result.append("new org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.Win(" + + param + "._partition().toList(),"); + Sort.processSortInfo(result, param + "._sortInfo()"); + result.append(");"); + result.append( + " }\n" + + "}).getFirst()"); + } +} + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowFuncArray.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowFuncArray.java new file mode 100644 index 00000000000..9eb56c258fb --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/ExtendWindowFuncArray.java @@ -0,0 +1,37 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; + +import org.eclipse.collections.api.list.ListIterable; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.compiled.generation.ProcessorContext; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; + +public class ExtendWindowFuncArray extends AbstractNative implements Native +{ + public ExtendWindowFuncArray() + { + super("extend_Relation_1___Window_1__FuncColSpecArray_1__Relation_1_"); + } + + @Override + public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) + { + StringBuilder result = ExtendWindowFunc.buildCode(transformedParams, s -> transformedParams.get(2) + "._funcSpecs().toList()"); + return result.toString(); + } +} + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/First.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/First.java new file mode 100644 index 00000000000..ae41853880c --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/First.java @@ -0,0 +1,27 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; + +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.Relation; +import org.finos.legend.pure.m3.execution.ExecutionSupport; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNativeFunctionGeneric; + +public class First extends AbstractNativeFunctionGeneric +{ + public First() + { + super("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.first", new Class[]{Relation.class, ExecutionSupport.class}, false, true, false, "first_Relation_1__T_1_"); + } +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/GroupBy.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/GroupBy.java index ba969ce0de6..a8d69d5124a 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/GroupBy.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/GroupBy.java @@ -38,22 +38,25 @@ public String build(CoreInstance topLevelElement, CoreInstance functionExpressio result.append(", "); result.append("Lists.mutable.with(" + transformedParams.get(2) + ")"); - processAggColSpec(transformedParams, result); + processAggColSpec(result, false); result.append(", es)"); return result.toString(); } - static void processAggColSpec(ListIterable transformedParams, StringBuilder result) + static void processAggColSpec(StringBuilder result, boolean twoArgs) { + String className = "AggColSpecTrans" + (twoArgs ? "2" : "1"); + String functionType = "org.eclipse.collections.api.block.function.Function" + (twoArgs ? "3" : "2"); + result.append(".collect("); - result.append("new DefendedFunction, org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.AggColSpecTrans>()\n" + + result.append("new DefendedFunction, org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation." + className + ">()\n" + "{\n" + " @Override\n" + - " public org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.AggColSpecTrans valueOf(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.AggColSpec c)\n" + + " public org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation." + className + " valueOf(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.AggColSpec c)\n" + " {\n"); - result.append("return new org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.AggColSpecTrans(c._name(),"); - result.append("(Function2)PureCompiledLambda.getPureFunction(c._map(),es),"); + result.append("return new org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation." + className + "(c._name(),"); + result.append("(" + functionType + ")PureCompiledLambda.getPureFunction(c._map(),es),"); result.append("(Function2)PureCompiledLambda.getPureFunction(c._reduce(),es),"); result.append("((org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType)c._reduce()._classifierGenericType()._typeArguments().toList().get(0)._rawType())._returnType()._rawType()._name());"); result.append(" }\n" + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/GroupByArray.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/GroupByArray.java index 6165b379771..1b925a4b760 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/GroupByArray.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/GroupByArray.java @@ -38,7 +38,7 @@ public String build(CoreInstance topLevelElement, CoreInstance functionExpressio result.append(transformedParams.get(1)); result.append(", "); result.append("Lists.mutable.withAll(" + transformedParams.get(2) + "._aggSpecs())"); - processAggColSpec(transformedParams, result); + processAggColSpec(result, false); result.append(", es)"); return result.toString(); } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Offset.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Offset.java new file mode 100644 index 00000000000..5118fbe7e6b --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Offset.java @@ -0,0 +1,28 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; + +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.Relation; +import org.finos.legend.pure.m3.execution.ExecutionSupport; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNativeFunctionGeneric; +import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.RowContainer; + +public class Offset extends AbstractNativeFunctionGeneric +{ + public Offset() + { + super("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.offset", new Class[]{Relation.class, RowContainer.class, Integer.class, ExecutionSupport.class}, false, true, false, "offset_Relation_1__T_1__Integer_1__T_1_"); + } +} diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Project.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Project.java index cca52593a64..a33007dc2d2 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Project.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Project.java @@ -38,7 +38,7 @@ public String build(CoreInstance topLevelElement, CoreInstance functionExpressio result.append("), "); result.append("Lists.mutable.withAll(" + transformedParams.get(1) + "._funcSpecs())"); - buildCollectFuncSpec(result); + buildCollectFuncSpec(result, false); // result.append("._names(), "); // result.append(transformedParams.get(1)); diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Sort.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Sort.java index 1e81721b347..3224b695fb1 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Sort.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Sort.java @@ -34,7 +34,14 @@ public String build(CoreInstance topLevelElement, CoreInstance functionExpressio result.append('('); result.append(transformedParams.get(0)); result.append(", "); - result.append("CompiledSupport.toPureCollection(" + transformedParams.get(1) + ")"); + processSortInfo(result, transformedParams.get(1)); + result.append(" , es)\n"); + return result.toString(); + } + + public static void processSortInfo(StringBuilder result, String param) + { + result.append("CompiledSupport.toPureCollection(" + param + ")"); result.append(".collect(new DefendedFunction, org.eclipse.collections.api.tuple.Pair>()\n" + "{\n" + " @Override\n" + @@ -42,7 +49,6 @@ public String build(CoreInstance topLevelElement, CoreInstance functionExpressio " {\n" + " return org.eclipse.collections.impl.tuple.Tuples.pair(it._direction(), it._column()._name());\n" + " }\n" + - "}) , es)\n"); - return result.toString(); + "})"); } } \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/NullRowContainer.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/NullRowContainer.java new file mode 100644 index 00000000000..977544c6d84 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/NullRowContainer.java @@ -0,0 +1,24 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared; + +public class NullRowContainer extends RowContainer +{ + @Override + public Object apply(String s) + { + return null; + } +} diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/RowContainer.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/RowContainer.java index 9268f2ad0eb..ca99c9eebf8 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/RowContainer.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/RowContainer.java @@ -21,12 +21,21 @@ public class RowContainer implements Function private int row; private TestTDSCompiled tds; + public RowContainer() + { + } + public RowContainer(TestTDSCompiled tds, int row) { this.tds = tds; this.row = row; } + public int getRow() + { + return row; + } + @Override public Object apply(String s) { diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/TestTDSCompiled.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/TestTDSCompiled.java index 8f045c00cab..3d2ec1d2e2c 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/TestTDSCompiled.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/TestTDSCompiled.java @@ -20,6 +20,7 @@ import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.api.map.MutableMap; import org.finos.legend.pure.m2.inlinedsl.tds.M2TDSPaths; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.Multiplicity; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Class; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; import org.finos.legend.pure.m3.navigation.ProcessorSupport; @@ -137,7 +138,7 @@ public TestTDSCompiled updateColumns(ProcessorSupport processorSupport) Class relationDatabaseAccessorType = (Class) processorSupport.package_getByUserPath(M2TDSPaths.TDS); GenericType genericType = (GenericType) processorSupport.type_wrapGenericType(relationDatabaseAccessorType); GenericType typeParam = (GenericType) processorSupport.newGenericType(null, relationDatabaseAccessorType, false); - MutableList columns = columnsOrdered.collect(c -> (CoreInstance) _Column.getColumnInstance(c,false, (GenericType) processorSupport.type_wrapGenericType(_Package.getByUserPath(convert(columnType.get(c)), processorSupport)), null, processorSupport)).toList(); + MutableList columns = columnsOrdered.collect(c -> (CoreInstance) _Column.getColumnInstance(c, false, (GenericType) processorSupport.type_wrapGenericType(_Package.getByUserPath(convert(columnType.get(c)), processorSupport)), (Multiplicity) org.finos.legend.pure.m3.navigation.multiplicity.Multiplicity.newMultiplicity(0, 1, processorSupport), null, processorSupport)).toList(); typeParam._rawType(_RelationType.build(columns, null, processorSupport)); genericType._typeArguments(Lists.mutable.with(typeParam)); this.classifierGenericType = genericType; diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/Win.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/Win.java new file mode 100644 index 00000000000..ba7b4cfd462 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/Win.java @@ -0,0 +1,46 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared; + +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.tuple.Pair; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpec; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enum; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortDirection; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortInfo; + +public class Win +{ + MutableList partition = Lists.mutable.empty(); + MutableList sorts = Lists.mutable.empty(); + + public Win(MutableList colSpec, RichIterable> sorts) + { + this.partition = colSpec; + this.sorts = sorts.toList().collect(c -> new SortInfo(c.getTwo(), SortDirection.valueOf(c.getOne()._name()))); + } + + public MutableList getPartition() + { + return partition; + } + + public MutableList getSorts() + { + return sorts; + } +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestFunctionTester.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/compiled/pure/TestFunctionTester.java similarity index 76% rename from legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestFunctionTester.java rename to legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/compiled/pure/TestFunctionTester.java index 2ee583d38fc..102a4112e6f 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestFunctionTester.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/compiled/pure/TestFunctionTester.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.pure.runtime.java.extension.relation; +package org.finos.legend.pure.runtime.java.extension.relation.compiled.pure; import org.finos.legend.pure.m3.execution.FunctionExecution; import org.finos.legend.pure.m3.tests.function.base.PureExpressionTest; @@ -43,42 +43,25 @@ protected static FunctionExecution getFunctionExecution() @org.junit.Test public void testFunction() { +// //-------------------------------------------------------------------- +// compileTestSource("fromString.pure", - "function test():Any[*]\n" + - "{" + - "print(#TDS\n" + - " value, str\n" + - " 1, a\n" + - " 3, ewe\n" + - " 4, qw\n" + - " #\n" + - " ->filter\n" + - " (\n" + - " x|$x.value == 1" + - " )->toString(),1);\n" + - "}"); + "function test():Any[*]\n" + + "{ " + + " let tds = #TDS\n" + + " id, name\n" + + " 1, George\n" + + " 2, Pierre\n" + + " 3, Sachin\n" + + " 4, David\n" + + " #;\n" + + "\n" + + " print(" + + " $tds->limit(2)->size()" + + " ,2);" + + "}\n"); this.execute("test():Any[*]"); runtime.delete("fromString.pure"); - -// //-------------------------------------------------------------------- -// -// compileTestSource("fromString.pure", -// "function test():Any[*]\n" + -// "{ " + -// " let tds = #TDS\n" + -// " id, name\n" + -// " 1, George\n" + -// " 2, Pierre\n" + -// " 3, Sachin\n" + -// " 4, David\n" + -// " #;\n" + -// "\n" + -// " print(" + -// " $tds->limit(2)->size()" + -// " ,2);" + -// "}\n"); -// this.execute("test():Any[*]"); -// runtime.delete("fromString.pure"); // // //-------------------------------------------------------------------- // diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java index f200da39732..d6e5d5cb232 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java @@ -29,6 +29,9 @@ import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Extend; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Filter; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.GroupBy; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.First; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Offset; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Last; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Join; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Limit; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Map; @@ -57,6 +60,15 @@ public RelationExtensionInterpreted() Tuples.pair("drop_Relation_1__Integer_1__Relation_1_", Drop::new), Tuples.pair("extend_Relation_1__FuncColSpec_1__Relation_1_", Extend::new), Tuples.pair("extend_Relation_1__FuncColSpecArray_1__Relation_1_", Extend::new), + Tuples.pair("extend_Relation_1__AggColSpec_1__Relation_1_", Extend::new), + Tuples.pair("extend_Relation_1__AggColSpecArray_1__Relation_1_", Extend::new), + Tuples.pair("extend_Relation_1___Window_1__AggColSpec_1__Relation_1_", Extend::new), + Tuples.pair("extend_Relation_1___Window_1__AggColSpecArray_1__Relation_1_", Extend::new), + Tuples.pair("extend_Relation_1___Window_1__FuncColSpec_1__Relation_1_", Extend::new), + Tuples.pair("extend_Relation_1___Window_1__FuncColSpecArray_1__Relation_1_", Extend::new), + Tuples.pair("first_Relation_1__T_1_", First::new), + Tuples.pair("last_Relation_1__T_1_", Last::new), + Tuples.pair("offset_Relation_1__T_1__Integer_1__T_1_", Offset::new), Tuples.pair("filter_Relation_1__Function_1__Relation_1_", Filter::new), Tuples.pair("map_Relation_1__Function_1__V_MANY_", Map::new), Tuples.pair("size_Relation_1__Integer_1_", Size::new), diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Extend.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Extend.java index 925e7a91378..2b073d703f8 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Extend.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Extend.java @@ -15,26 +15,35 @@ package org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives; import io.deephaven.csv.parsers.DataType; +import org.eclipse.collections.api.block.procedure.Procedure2; import org.eclipse.collections.api.list.FixedSizeList; import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.api.tuple.Pair; import org.eclipse.collections.impl.factory.Lists; import org.finos.legend.pure.m3.compiler.Context; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunctionCoreInstanceWrapper; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.AggColSpec; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.AggColSpecArray; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.FuncColSpec; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.FuncColSpecArray; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Type; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; import org.finos.legend.pure.m3.exception.PureExecutionException; import org.finos.legend.pure.m3.navigation.*; import org.finos.legend.pure.m3.navigation._package._Package; import org.finos.legend.pure.m4.ModelRepository; import org.finos.legend.pure.m4.coreinstance.CoreInstance; -import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.Shared; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.AggregationShared; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSCoreInstance; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSWithCursorCoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.ColumnValue; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortDirection; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortInfo; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; import org.finos.legend.pure.runtime.java.interpreted.ExecutionSupport; import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted; @@ -42,9 +51,11 @@ import org.finos.legend.pure.runtime.java.interpreted.natives.InstantiationContext; import org.finos.legend.pure.runtime.java.interpreted.profiler.Profiler; +import java.lang.reflect.Executable; +import java.util.Arrays; import java.util.Stack; -public class Extend extends Shared +public class Extend extends AggregationShared { public Extend(FunctionExecutionInterpreted functionExecution, ModelRepository repository) { @@ -54,85 +65,166 @@ public Extend(FunctionExecutionInterpreted functionExecution, ModelRepository re @Override public CoreInstance execute(ListIterable params, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, Context context, ProcessorSupport processorSupport) throws PureExecutionException { - CoreInstance returnGenericType = getReturnGenericType(resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, processorSupport); + try + { + CoreInstance returnGenericType = getReturnGenericType(resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, processorSupport); - TestTDS tds = getTDS(params, 0, processorSupport); + TestTDS tds = getTDS(params, 0, processorSupport); - RelationType relationType = getRelationType(params, 0); + RelationType relationType = getRelationType(params, 0); + GenericType sourceRelationType = (GenericType) params.get(0).getValueForMetaPropertyToOne("genericType"); - CoreInstance extendFunction = Instance.getValueForMetaPropertyToOneResolved(params.get(1), M3Properties.values, processorSupport); + CoreInstance secondParameter = Instance.getValueForMetaPropertyToOneResolved(params.get(1), M3Properties.values, processorSupport); + + TestTDS result; + if (secondParameter instanceof FuncColSpec) + { + result = tds.addColumn(processFuncColSpec(tds.wrapFullTDS(), (FuncColSpec) secondParameter, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, (GenericType) params.get(0).getValueForMetaPropertyToOne("genericType"), false)); + } + else if (secondParameter instanceof FuncColSpecArray) + { + result = ((FuncColSpecArray) secondParameter)._funcSpecs().injectInto( + tds, + (a, funcColSpec) -> a.addColumn(processFuncColSpec(tds.wrapFullTDS(), funcColSpec, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, sourceRelationType, false)) + ); + } + else if (secondParameter instanceof AggColSpec) + { + Pair>> source = tds.wrapFullTDS(); + result = tds.addColumn(processOneAggColSpec(source, (AggColSpec) secondParameter, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, false, false, null)); + } + else if (secondParameter instanceof AggColSpecArray) + { + Pair>> source = tds.wrapFullTDS(); + result = ((AggColSpecArray) secondParameter)._aggSpecs().injectInto( + source.getOne(), + (a, aggColSpec) -> a.addColumn(processOneAggColSpec(source, aggColSpec, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, false, false, null)) + ); + } + else if (Instance.instanceOf(secondParameter, "meta::pure::functions::relation::_Window", processorSupport)) + { + MutableList partitionIds = secondParameter.getValueForMetaPropertyToMany("partition").collect(PrimitiveUtilities::getStringValue).toList(); + Pair>> source = partitionIds.isEmpty() ? tds.wrapFullTDS() : tds.sort(partitionIds.collect(c -> new SortInfo(c, SortDirection.ASC))); + + ListIterable sortInfos = secondParameter.getValueForMetaPropertyToMany("sortInfo"); + final Pair>> sortedPartitions = TestTDS.sortPartitions(Sort.getSortInfos(sortInfos, processorSupport).toList(), source); + + CoreInstance thirdParameter = Instance.getValueForMetaPropertyToOneResolved(params.get(2), M3Properties.values, processorSupport); + if (thirdParameter instanceof AggColSpec) + { + result = sortedPartitions.getOne().addColumn(processOneAggColSpec(sortedPartitions, (AggColSpec) thirdParameter, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, false, true, sourceRelationType)); + } + else if (thirdParameter instanceof AggColSpecArray) + { + result = ((AggColSpecArray) thirdParameter)._aggSpecs().injectInto( + sortedPartitions.getOne(), + (a, aggColSpec) -> a.addColumn(processOneAggColSpec(sortedPartitions, aggColSpec, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, false, true, sourceRelationType)) + ); + } + else if (thirdParameter instanceof FuncColSpec) + { + result = sortedPartitions.getOne().addColumn(processFuncColSpec(sortedPartitions, (FuncColSpec) thirdParameter, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, sourceRelationType, true)); + } + else if (thirdParameter instanceof FuncColSpecArray) + { + result = ((FuncColSpecArray) thirdParameter)._funcSpecs().injectInto( + sortedPartitions.getOne(), + (a, funcColSpec) -> a.addColumn(processFuncColSpec(sortedPartitions, funcColSpec, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, sourceRelationType, true)) + ); + } + else + { + throw new RuntimeException("Not possible"); + } + } + else + { + throw new RuntimeException("Not possible"); + } + return ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(result, returnGenericType, repository, processorSupport), false, processorSupport); - TestTDS result; - if (extendFunction instanceof FuncColSpec) - { - result = processFuncColSpec(tds, - (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(extendFunction.getValueForMetaPropertyToOne(M3Properties.function)), - extendFunction.getValueForMetaPropertyToOne(M3Properties.name).getName(), - resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType - ); - } - else if (extendFunction instanceof FuncColSpecArray) - { - result = ((FuncColSpecArray) extendFunction)._funcSpecs().injectInto(tds, (a, b) -> - processFuncColSpec(a, - (LambdaFunction) b._function(), - b._name(), - resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType - )); } - else + catch (Exception e) { - throw new RuntimeException("Not possible"); + e.printStackTrace(); + throw e; } - - return ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(result, returnGenericType, repository, processorSupport), false, processorSupport); } - private TestTDS processFuncColSpec(TestTDS tds, LambdaFunction lambdaFunction, String name, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, RelationType relationType) + private ColumnValue processFuncColSpec(Pair>> source, FuncColSpec funcColSpec, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, GenericType relationType, boolean twoParamsFunc) { + LambdaFunction lambdaFunction = (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(funcColSpec.getValueForMetaPropertyToOne(M3Properties.function)); + String name = funcColSpec.getValueForMetaPropertyToOne(M3Properties.name).getName(); + VariableContext evalVarContext = this.getParentOrEmptyVariableContextForLambda(variableContext, lambdaFunction); - FixedSizeList parameters = Lists.fixedSize.with((CoreInstance) null); Type type = ((FunctionType) lambdaFunction._classifierGenericType()._typeArguments().getFirst()._rawType())._returnType()._rawType(); - Object res = null; - DataType resType = null; if (type == _Package.getByUserPath("String", processorSupport)) { - String[] resStr = new String[(int) tds.getRowCount()]; - for (int i = 0; i < tds.getRowCount(); i++) - { - parameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(tds, i, "", null, relationType, -1, repository, false), true, processorSupport)); - CoreInstance newValue = this.functionExecution.executeFunction(false, lambdaFunction, parameters, resolvedTypeParameters, resolvedMultiplicityParameters, evalVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport); - resStr[i] = PrimitiveUtilities.getStringValue(newValue.getValueForMetaPropertyToOne("values")); - } - res = resStr; - resType = DataType.STRING; + String[] finalRes = new String[(int) source.getOne().getRowCount()]; + processOneColumn(source, lambdaFunction, (j, val) -> finalRes[j] = val == null ? null : PrimitiveUtilities.getStringValue(val), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, evalVarContext, twoParamsFunc); + return new ColumnValue(name, DataType.STRING, finalRes); } else if (type == _Package.getByUserPath("Integer", processorSupport)) { - int[] resInt = new int[(int) tds.getRowCount()]; - for (int i = 0; i < tds.getRowCount(); i++) + int[] finalRes = new int[(int) source.getOne().getRowCount()]; + boolean[] nulls = new boolean[(int) source.getOne().getRowCount()]; + Arrays.fill(nulls, Boolean.FALSE); + processOneColumn(source, lambdaFunction, (j, val) -> processWithNull(j, val, nulls, () -> finalRes[j] = PrimitiveUtilities.getIntegerValue(val).intValue()), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, evalVarContext, twoParamsFunc); + return new ColumnValue(name, DataType.INT, finalRes, nulls); + } + else if (type == _Package.getByUserPath("Float", processorSupport)) + { + double[] finalRes = new double[(int) source.getOne().getRowCount()]; + boolean[] nulls = new boolean[(int) source.getOne().getRowCount()]; + Arrays.fill(nulls, Boolean.FALSE); + processOneColumn(source, lambdaFunction, (j, val) -> processWithNull(j, val, nulls, () -> finalRes[j] = PrimitiveUtilities.getFloatValue(val).doubleValue()), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, evalVarContext, twoParamsFunc); + return new ColumnValue(name, DataType.DOUBLE, finalRes, nulls); + } + else + { + throw new RuntimeException("The type " + type._name() + " is not supported yet!"); + } + } + + private interface Proc + { + void invoke(); + } + + private void processWithNull(Integer j, CoreInstance val, boolean[] nulls, Proc p) + { + { + if (val == null) { - parameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(tds, i, "", null, relationType, -1, repository, false), true, processorSupport)); - CoreInstance newValue = this.functionExecution.executeFunction(false, lambdaFunction, parameters, resolvedTypeParameters, resolvedMultiplicityParameters, evalVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport); - resInt[i] = PrimitiveUtilities.getIntegerValue(newValue.getValueForMetaPropertyToOne("values")).intValue(); + nulls[j] = true; + } + else + { + p.invoke(); } - res = resInt; - resType = DataType.INT; } - else if (type == _Package.getByUserPath("Float", processorSupport)) + } + + private void processOneColumn(Pair>> source, LambdaFunction lambdaFunction, Procedure2 setter, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, GenericType relationType, VariableContext evalVarContext, boolean twoParamsFunc) + { + FixedSizeList parameters = twoParamsFunc ? Lists.fixedSize.with((CoreInstance) null, (CoreInstance) null) : Lists.fixedSize.with((CoreInstance) null); + int k = 0; + for (int j = 0; j < source.getTwo().size(); j++) { - double[] resDouble = new double[(int) tds.getRowCount()]; - for (int i = 0; i < tds.getRowCount(); i++) + Pair r = source.getTwo().get(j); + TestTDS sourceTDS = source.getOne().slice(r.getOne(), r.getTwo()); + for (int i = 0; i < r.getTwo() - r.getOne(); i++) { - parameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(tds, i, "", null, relationType, -1, repository, false), true, processorSupport)); + if (twoParamsFunc) + { + parameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(sourceTDS, relationType, repository, processorSupport), false, processorSupport)); + } + parameters.set(twoParamsFunc ? 1 : 0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(source.getOne(), i, "", null, relationType, -1, repository, false), false, processorSupport)); CoreInstance newValue = this.functionExecution.executeFunction(false, lambdaFunction, parameters, resolvedTypeParameters, resolvedMultiplicityParameters, evalVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport); - resDouble[i] = PrimitiveUtilities.getFloatValue(newValue.getValueForMetaPropertyToOne("values")).doubleValue(); + setter.value(k++, newValue.getValueForMetaPropertyToOne("values")); } - res = resDouble; - resType = DataType.DOUBLE; } - return tds.addColumn(name, resType, res); } } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/First.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/First.java new file mode 100644 index 00000000000..eee32128feb --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/First.java @@ -0,0 +1,51 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives; + +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.map.MutableMap; +import org.finos.legend.pure.m3.compiler.Context; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; +import org.finos.legend.pure.m3.exception.PureExecutionException; +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.navigation.ValueSpecificationBootstrap; +import org.finos.legend.pure.m4.ModelRepository; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.Shared; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSWithCursorCoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; +import org.finos.legend.pure.runtime.java.interpreted.ExecutionSupport; +import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted; +import org.finos.legend.pure.runtime.java.interpreted.VariableContext; +import org.finos.legend.pure.runtime.java.interpreted.natives.InstantiationContext; +import org.finos.legend.pure.runtime.java.interpreted.profiler.Profiler; + +import java.util.Stack; + +public class First extends Shared +{ + public First(FunctionExecutionInterpreted functionExecution, ModelRepository repository) + { + super(functionExecution, repository); + } + + @Override + public CoreInstance execute(ListIterable params, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, Context context, ProcessorSupport processorSupport) throws PureExecutionException + { + TestTDS tds = getTDS(params, 0, processorSupport); + RelationType relationType = getRelationType(params, 0); + return ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(tds, 0, "", null, relationType, -1, repository, false), true, processorSupport); + } +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/GroupBy.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/GroupBy.java index 8b953d0d69d..1808069803e 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/GroupBy.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/GroupBy.java @@ -14,28 +14,23 @@ package org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives; -import io.deephaven.csv.parsers.DataType; -import org.eclipse.collections.api.block.procedure.Procedure2; -import org.eclipse.collections.api.list.FixedSizeList; import org.eclipse.collections.api.list.ListIterable; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.api.map.MutableMap; import org.eclipse.collections.api.tuple.Pair; -import org.eclipse.collections.impl.factory.Lists; import org.finos.legend.pure.m3.compiler.Context; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunctionCoreInstanceWrapper; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.*; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Type; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.AggColSpec; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.AggColSpecArray; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; import org.finos.legend.pure.m3.exception.PureExecutionException; -import org.finos.legend.pure.m3.navigation.*; -import org.finos.legend.pure.m3.navigation._package._Package; +import org.finos.legend.pure.m3.navigation.Instance; +import org.finos.legend.pure.m3.navigation.M3Properties; +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.navigation.ValueSpecificationBootstrap; import org.finos.legend.pure.m4.ModelRepository; import org.finos.legend.pure.m4.coreinstance.CoreInstance; -import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.Shared; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.AggregationShared; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSCoreInstance; -import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSWithCursorCoreInstance; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortDirection; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortInfo; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; @@ -47,7 +42,7 @@ import java.util.Stack; -public class GroupBy extends Shared +public class GroupBy extends AggregationShared { public GroupBy(FunctionExecutionInterpreted functionExecution, ModelRepository repository) { @@ -63,30 +58,23 @@ public CoreInstance execute(ListIterable params, Stack relationType = getRelationType(params, 0); + // Col Ids Object cols = Instance.getValueForMetaPropertyToOneResolved(params.get(1), M3Properties.values, processorSupport); - ListIterable ids; - if (cols instanceof ColSpec) - { - ids = Lists.mutable.with(((ColSpec) cols)._name()); - } - else if (cols instanceof ColSpecArray) - { - ids = ((ColSpecArray) cols)._names().collect(c -> (String) c).toList(); - } - else - { - throw new RuntimeException("Not Possible"); - } + ListIterable ids = getColumnIds(cols); - CoreInstance aggColSpec = Instance.getValueForMetaPropertyToOneResolved(params.get(2), M3Properties.values, processorSupport); - TestTDS result; - if (aggColSpec instanceof AggColSpec) + // Build TDS + Pair>> orderedSource = tds.sort(ids.collect(c -> new SortInfo(c, SortDirection.ASC))); + TestTDS result = orderedSource.getOne()._distinct(orderedSource.getTwo()); + + // Aggregations + CoreInstance aggSpec = Instance.getValueForMetaPropertyToOneResolved(params.get(2), M3Properties.values, processorSupport); + if (aggSpec instanceof AggColSpec) { - result = processOneAggColSpec(tds, null, ids, aggColSpec, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType); + result.addColumn(processOneAggColSpec(orderedSource, (AggColSpec) aggSpec, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, true, false, null)); } - else if (aggColSpec instanceof AggColSpecArray) + else if (aggSpec instanceof AggColSpecArray) { - result = ((AggColSpecArray) aggColSpec)._aggSpecs().injectInto(null, (a, b) -> processOneAggColSpec(tds, a, ids, b, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType)); + result = ((AggColSpecArray) aggSpec)._aggSpecs().injectInto(result, (accResult, aggColSpec) -> accResult.addColumn(processOneAggColSpec(orderedSource, aggColSpec, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, true, false, null))); } else { @@ -96,66 +84,4 @@ else if (aggColSpec instanceof AggColSpecArray) return ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(result, returnGenericType, repository, processorSupport), false, processorSupport); } - private TestTDS processOneAggColSpec(TestTDS tds, TestTDS existing, ListIterable ids, CoreInstance aggColSpec, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, RelationType relationType) - { - String name = aggColSpec.getValueForMetaPropertyToOne("name").getName(); - LambdaFunction mapF = (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(aggColSpec.getValueForMetaPropertyToOne("map")); - LambdaFunction reduceF = (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(aggColSpec.getValueForMetaPropertyToOne("reduce")); - - VariableContext mapFVarContext = this.getParentOrEmptyVariableContextForLambda(variableContext, mapF); - VariableContext reduceFVarContext = this.getParentOrEmptyVariableContextForLambda(variableContext, reduceF); - - Type type = ((FunctionType) reduceF._classifierGenericType()._typeArguments().getFirst()._rawType())._returnType()._rawType(); - - Pair>> res = tds.sort(ids.collect(c -> new SortInfo(c, SortDirection.ASC))); - - FixedSizeList parameters = Lists.fixedSize.with((CoreInstance) null); - - int size = res.getTwo().size(); - DataType resType = null; - Object _finalRes = null; - if (type == _Package.getByUserPath("String", processorSupport)) - { - String[] finalRes = new String[size]; - performAggregation(res, mapF, reduceF, (j, val) -> finalRes[j] = PrimitiveUtilities.getStringValue(val), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, size, parameters, mapFVarContext, reduceFVarContext); - resType = DataType.STRING; - _finalRes = finalRes; - } - else if (type == _Package.getByUserPath("Integer", processorSupport)) - { - int[] finalRes = new int[size]; - performAggregation(res, mapF, reduceF, (j, val) -> finalRes[j] = PrimitiveUtilities.getIntegerValue(val).intValue(), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, size, parameters, mapFVarContext, reduceFVarContext); - resType = DataType.INT; - _finalRes = finalRes; - } - else if (type == _Package.getByUserPath("Float", processorSupport) || type == _Package.getByUserPath("Number", processorSupport)) - { - double[] finalRes = new double[size]; - performAggregation(res, mapF, reduceF, (j, val) -> finalRes[j] = PrimitiveUtilities.getFloatValue(val).doubleValue(), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, size, parameters, mapFVarContext, reduceFVarContext); - resType = DataType.FLOAT; - _finalRes = finalRes; - } - else - { - throw new RuntimeException("The type " + type._name() + " is not supported yet!"); - } - return existing == null ? res.getOne()._distinct(res.getTwo()).addColumn(name, resType, _finalRes) : existing.addColumn(name, resType, _finalRes); - } - - private void performAggregation(Pair>> res, LambdaFunction mapF, LambdaFunction reduceF, Procedure2 setter, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, RelationType relationType, int size, FixedSizeList parameters, VariableContext mapFVarContext, VariableContext reduceFVarContext) - { - for (int j = 0; j < size; j++) - { - Pair r = res.getTwo().get(j); - MutableList subList = Lists.mutable.empty(); - for (int i = r.getOne(); i < r.getTwo(); i++) - { - parameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(res.getOne(), i, "", null, relationType, -1, repository, false), true, processorSupport)); - subList.add(this.functionExecution.executeFunction(false, mapF, parameters, resolvedTypeParameters, resolvedMultiplicityParameters, mapFVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport).getValueForMetaPropertyToOne("values")); - } - parameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(subList, true, processorSupport)); - CoreInstance re = this.functionExecution.executeFunction(false, reduceF, parameters, resolvedTypeParameters, resolvedMultiplicityParameters, reduceFVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport); - setter.value(j, re.getValueForMetaPropertyToOne("values")); - } - } } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Last.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Last.java new file mode 100644 index 00000000000..e623387d9c8 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Last.java @@ -0,0 +1,52 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives; + +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.map.MutableMap; +import org.finos.legend.pure.m3.compiler.Context; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; +import org.finos.legend.pure.m3.exception.PureExecutionException; +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.navigation.ValueSpecificationBootstrap; +import org.finos.legend.pure.m4.ModelRepository; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.Shared; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSWithCursorCoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; +import org.finos.legend.pure.runtime.java.interpreted.ExecutionSupport; +import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted; +import org.finos.legend.pure.runtime.java.interpreted.VariableContext; +import org.finos.legend.pure.runtime.java.interpreted.natives.InstantiationContext; +import org.finos.legend.pure.runtime.java.interpreted.profiler.Profiler; + +import java.util.Stack; + +public class Last extends Shared +{ + public Last(FunctionExecutionInterpreted functionExecution, ModelRepository repository) + { + super(functionExecution, repository); + } + + @Override + public CoreInstance execute(ListIterable params, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, Context context, ProcessorSupport processorSupport) throws PureExecutionException + { + TestTDS tds = getTDS(params, 0, processorSupport); + RelationType relationType = getRelationType(params, 0); + return ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(tds, (int)tds.getRowCount() - 1, "", null, relationType, -1, repository, false), true, processorSupport); + } +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Offset.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Offset.java new file mode 100644 index 00000000000..627972ddebf --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Offset.java @@ -0,0 +1,67 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives; + +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.map.MutableMap; +import org.finos.legend.pure.m3.compiler.Context; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; +import org.finos.legend.pure.m3.exception.PureExecutionException; +import org.finos.legend.pure.m3.navigation.Instance; +import org.finos.legend.pure.m3.navigation.M3Properties; +import org.finos.legend.pure.m3.navigation.PrimitiveUtilities; +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.navigation.ValueSpecificationBootstrap; +import org.finos.legend.pure.m4.ModelRepository; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.Shared; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSWithCursorCoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; +import org.finos.legend.pure.runtime.java.interpreted.ExecutionSupport; +import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted; +import org.finos.legend.pure.runtime.java.interpreted.VariableContext; +import org.finos.legend.pure.runtime.java.interpreted.natives.InstantiationContext; +import org.finos.legend.pure.runtime.java.interpreted.profiler.Profiler; + +import java.util.Stack; + +public class Offset extends Shared +{ + public Offset(FunctionExecutionInterpreted functionExecution, ModelRepository repository) + { + super(functionExecution, repository); + } + + @Override + public CoreInstance execute(ListIterable params, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, Context context, ProcessorSupport processorSupport) throws PureExecutionException + { + TestTDS tds = getTDS(params, 0, processorSupport); + TestTDS nullTDS = tds.newNullTDS(); + + RelationType relationType = getRelationType(params, 0); + + int currentRow = (((TDSWithCursorCoreInstance) params.get(1).getValueForMetaPropertyToOne("values")).getCurrentRow()); + int offset = PrimitiveUtilities.getIntegerValue(Instance.getValueForMetaPropertyToOneResolved(params.get(2), M3Properties.values, processorSupport)).intValue(); + int newRow = currentRow + offset; + if (newRow < 0 || newRow >= tds.getRowCount()) + { + return ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(nullTDS, 0, "", null, relationType, -1, repository, false), true, processorSupport); + } + else + { + return ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(tds, newRow, "", null, relationType, -1, repository, false), true, processorSupport); + } + } +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Sort.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Sort.java index 216443427b3..1259602d49f 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Sort.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Sort.java @@ -49,14 +49,19 @@ public CoreInstance execute(ListIterable params, Stack sortInfo = Instance.getValueForMetaPropertyToManyResolved(params.get(1), M3Properties.values, processorSupport); + ListIterable sortInfos = getSortInfos(Instance.getValueForMetaPropertyToManyResolved(params.get(1), M3Properties.values, processorSupport), processorSupport); + return ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(tds.sort(sortInfos).getOne(), returnGenericType, repository, processorSupport), false, processorSupport); + } + + public static ListIterable getSortInfos(ListIterable sortInfo, ProcessorSupport processorSupport) + { ListIterable sortInfos = sortInfo.collect(c -> { String name = c.getValueForMetaPropertyToOne("column").getValueForMetaPropertyToOne("name").getName(); SortDirection direction = SortDirection.valueOf(c.getValueForMetaPropertyToOne("direction").getName()); return new SortInfo(name, direction); }); - return ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(tds.sort(sortInfos).getOne(), returnGenericType, repository, processorSupport), false, processorSupport); + return sortInfos; } } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/AggregationShared.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/AggregationShared.java new file mode 100644 index 00000000000..07adc4cd995 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/AggregationShared.java @@ -0,0 +1,150 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared; + +import io.deephaven.csv.parsers.DataType; +import org.eclipse.collections.api.block.procedure.Procedure2; +import org.eclipse.collections.api.list.FixedSizeList; +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.api.tuple.Pair; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunctionCoreInstanceWrapper; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.AggColSpec; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpec; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpecArray; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Type; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; +import org.finos.legend.pure.m3.navigation.PrimitiveUtilities; +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.navigation.ValueSpecificationBootstrap; +import org.finos.legend.pure.m3.navigation._package._Package; +import org.finos.legend.pure.m4.ModelRepository; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.ColumnValue; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; +import org.finos.legend.pure.runtime.java.interpreted.ExecutionSupport; +import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted; +import org.finos.legend.pure.runtime.java.interpreted.VariableContext; +import org.finos.legend.pure.runtime.java.interpreted.natives.InstantiationContext; +import org.finos.legend.pure.runtime.java.interpreted.profiler.Profiler; + +import java.util.Stack; + +public abstract class AggregationShared extends Shared +{ + public AggregationShared(FunctionExecutionInterpreted functionExecution, ModelRepository repository) + { + super(functionExecution, repository); + } + + protected ColumnValue processOneAggColSpec(Pair>> aggregationScope, AggColSpec aggColSpec, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, RelationType relationType, boolean compress, boolean twoParamsFunc, GenericType sourceTDSType) + { + String name = aggColSpec.getValueForMetaPropertyToOne("name").getName(); + LambdaFunction mapF = (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(aggColSpec.getValueForMetaPropertyToOne("map")); + LambdaFunction reduceF = (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(aggColSpec.getValueForMetaPropertyToOne("reduce")); + + VariableContext mapFVarContext = this.getParentOrEmptyVariableContextForLambda(variableContext, mapF); + VariableContext reduceFVarContext = this.getParentOrEmptyVariableContextForLambda(variableContext, reduceF); + + Type type = ((FunctionType) reduceF._classifierGenericType()._typeArguments().getFirst()._rawType())._returnType()._rawType(); + + int size = compress ? aggregationScope.getTwo().size() : (int) aggregationScope.getOne().getRowCount(); + if (type == _Package.getByUserPath("String", processorSupport)) + { + String[] finalRes = new String[size]; + performAggregation(aggregationScope, mapF, reduceF, (j, val) -> finalRes[j] = PrimitiveUtilities.getStringValue(val), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, mapFVarContext, reduceFVarContext, compress, twoParamsFunc, sourceTDSType); + return new ColumnValue(name, DataType.STRING, finalRes); + } + else if (type == _Package.getByUserPath("Integer", processorSupport)) + { + int[] finalRes = new int[size]; + performAggregation(aggregationScope, mapF, reduceF, (j, val) -> finalRes[j] = PrimitiveUtilities.getIntegerValue(val).intValue(), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, mapFVarContext, reduceFVarContext, compress, twoParamsFunc, sourceTDSType); + return new ColumnValue(name, DataType.INT, finalRes); + } + else if (type == _Package.getByUserPath("Float", processorSupport) || type == _Package.getByUserPath("Number", processorSupport)) + { + double[] finalRes = new double[size]; + performAggregation(aggregationScope, mapF, reduceF, (j, val) -> finalRes[j] = PrimitiveUtilities.getFloatValue(val).doubleValue(), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, mapFVarContext, reduceFVarContext, compress, twoParamsFunc, sourceTDSType); + return new ColumnValue(name, DataType.FLOAT, finalRes); + } + else + { + throw new RuntimeException("The type " + type._name() + " is not supported yet!"); + } + } + + protected void performAggregation(Pair>> orderedSource, LambdaFunction mapF, LambdaFunction reduceF, Procedure2 setter, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, RelationType relationType, VariableContext mapFVarContext, VariableContext reduceFVarContext, boolean compress, boolean twoParamFunc, GenericType sourceTDSType) + { + FixedSizeList mapParameters = twoParamFunc ? Lists.fixedSize.with((CoreInstance) null, (CoreInstance) null) : Lists.fixedSize.with((CoreInstance) null); + FixedSizeList reduceParameters = Lists.fixedSize.with((CoreInstance) null); + int size = orderedSource.getTwo().size(); + int uncompressedCursor = 0; + for (int j = 0; j < size; j++) + { + Pair r = orderedSource.getTwo().get(j); + MutableList subList = Lists.mutable.empty(); + TestTDS sourceTDS = orderedSource.getOne().slice(r.getOne(), r.getTwo()); + for (int i = 0; i < r.getTwo() - r.getOne(); i++) + { + if (twoParamFunc) + { + mapParameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(sourceTDS, sourceTDSType, repository, processorSupport), false, processorSupport)); + } + mapParameters.set(twoParamFunc ? 1 : 0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(sourceTDS, i, "", null, relationType, -1, this.repository, false), true, processorSupport)); + CoreInstance oneRes = this.functionExecution.executeFunction(false, mapF, mapParameters, resolvedTypeParameters, resolvedMultiplicityParameters, mapFVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport).getValueForMetaPropertyToOne("values"); + if (oneRes != null) + { + subList.add(oneRes); + } + } + reduceParameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(subList, true, processorSupport)); + CoreInstance re = this.functionExecution.executeFunction(false, reduceF, reduceParameters, resolvedTypeParameters, resolvedMultiplicityParameters, reduceFVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport); + if (compress) + { + setter.value(j, re.getValueForMetaPropertyToOne("values")); + } + else + { + for (int i = r.getOne(); i < r.getTwo(); i++) + { + setter.value(uncompressedCursor++, re.getValueForMetaPropertyToOne("values")); + } + } + } + } + + protected static ListIterable getColumnIds(Object cols) + { + ListIterable ids; + if (cols instanceof ColSpec) + { + ids = Lists.mutable.with(((ColSpec) cols)._name()); + } + else if (cols instanceof ColSpecArray) + { + ids = ((ColSpecArray) cols)._names().collect(c -> (String) c).toList(); + } + else + { + throw new RuntimeException("Not Possible"); + } + return ids; + } +} diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/TDSWithCursorCoreInstance.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/TDSWithCursorCoreInstance.java index 2c43d13309c..23af2553909 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/TDSWithCursorCoreInstance.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/TDSWithCursorCoreInstance.java @@ -36,4 +36,9 @@ public CoreInstance getValue(String name) { return tds.getValueAsCoreInstance(name, currentRow); } + + public int getCurrentRow() + { + return currentRow; + } } \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/interpreted/pure/TestFunctionTester.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/interpreted/pure/TestFunctionTester.java new file mode 100644 index 00000000000..9b96deac7d7 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/interpreted/pure/TestFunctionTester.java @@ -0,0 +1,232 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.relation.interpreted.pure; + +import org.finos.legend.pure.m3.execution.FunctionExecution; +import org.finos.legend.pure.m3.tests.function.base.PureExpressionTest; +import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted; +import org.junit.After; +import org.junit.BeforeClass; + +public class TestFunctionTester extends PureExpressionTest +{ + @BeforeClass + public static void setUp() + { + setUpRuntime(getFunctionExecution()); + } + + @After + public void cleanRuntime() + { + runtime.delete("fromString.pure"); + } + + protected static FunctionExecution getFunctionExecution() + { + FunctionExecutionInterpreted fi = new FunctionExecutionInterpreted(); + fi.getConsole().enable(); + return fi; + } + + @org.junit.Test + public void testFunction() + { +// compileTestSource("fromString.pure", +// "function test():Any[*]\n" + +// "{" + +// "print(#TDS\n" + +// " id, grp, name\n" + +// " 1, 2, A\n" + +// " 2, 1, B\n" + +// " 3, 3, C\n" + +// " 4, 4, D\n" + +// " 5, 2, E\n" + +// " 6, 1, F\n" + +// " 7, 3, G\n" + +// " 8, 1, H\n" + +// " 9, 5, I\n" + +// " 10, 0, J\n" + +// " #->groupBy(~grp, ~[newCol : x | $x.name : y | $y->joinStrings(''), YoCol : x | $x.id : y | $y->plus()])->toString(),1);\n" + +// "}"); +// this.execute("test():Any[*]"); +// runtime.delete("fromString.pure"); +// +// +// compileTestSource("fromString.pure", +// "function test():Any[*]\n" + +// "{" + +// "print(#TDS\n" + +// " val, str\n" + +// " 1, a\n" + +// " 3, ewe\n" + +// " 4, qw\n" + +// " 5, wwe\n" + +// " 6, weq\n" + +// " #->extend(~[name:c|$c.val->toOne() + 1, other:x|$x.str->toOne()+'_ext'])->toString(),1);\n" + +// "}"); +// this.execute("test():Any[*]"); +// runtime.delete("fromString.pure"); + +// compileTestSource("fromString.pure", +// "function test():Any[*]\n" + +// "{" + +// "print(#TDS\n" + +// " id, grp, name\n" + +// " 1, 2, A\n" + +// " 2, 1, B\n" + +// " 3, 3, C\n" + +// " 4, 4, D\n" + +// " 5, 2, E\n" + +// " 6, 1, F\n" + +// " 7, 3, G\n" + +// " 8, 1, H\n" + +// " 9, 5, I\n" + +// " 10, 0, J\n" + +// " #" + +// //" ->extend(~newOne:{x|$x.id}:y|$y->plus())" + +// //" ->extend(win(~grp), ~t:{w,z|$z.id}:y|$y->plus())" + +// " ->extend(win(~grp, [~id->descending()]), ~te:{w,z|$w->lead($z).id})" + +// " ->toString(),1);\n" + +// "}"); +// this.execute("test():Any[*]"); +// runtime.delete("fromString.pure"); + + + compileTestSource("fromString.pure", + "function test():Any[*]\n" + + "{" + + "print( #TDS\n" + + " id, grp, name\n" + + " 1, 2, A\n" + + " 2, 1, B\n" + + " 3, 3, C\n" + + " 4, 4, D\n" + + " 5, 2, E\n" + + " 6, 1, F\n" + + " 7, 3, G\n" + + " 8, 1, H\n" + + " 9, 5, I\n" + + " 10, 0, J\n" + + " #->extend(over(~grp), ~newCol:{w,c|$c.id}:y|$y->plus())" + + " ->toString(),1);\n" + + "}"); + this.execute("test():Any[*]"); + runtime.delete("fromString.pure"); + +// //-------------------------------------------------------------------- +// +// compileTestSource("fromString.pure", +// "function test():Any[*]\n" + +// "{ " + +// " let tds = #TDS\n" + +// " id, name\n" + +// " 1, George\n" + +// " 2, Pierre\n" + +// " 3, Sachin\n" + +// " 4, David\n" + +// " #;\n" + +// "\n" + +// " print(" + +// " $tds->limit(2)->size()" + +// " ,2);" + +// "}\n"); +// this.execute("test():Any[*]"); +// runtime.delete("fromString.pure"); +// +// //-------------------------------------------------------------------- +// +// compileTestSource("fromString.pure", +// "function test():Any[*]\n" + +// "{ " + +// " let tds = #TDS\n" + +// " id, name\n" + +// " 1, George\n" + +// " 2, Pierre\n" + +// " 3, Sachin\n" + +// " 4, David\n" + +// " #;\n" + +// "\n" + +// " print(" + +// " $tds->drop(3)->size()" + +// " ,2);" + +// "}\n"); +// this.execute("test():Any[*]"); +// runtime.delete("fromString.pure"); +// +// //-------------------------------------------------------------------- +// +// compileTestSource("fromString.pure", +// "function test():Any[*]\n" + +// "{ " + +// " let tds = #TDS\n" + +// " id, name\n" + +// " 1, George\n" + +// " 2, Pierre\n" + +// " 3, Sachin\n" + +// " 4, David\n" + +// " #;\n" + +// "\n" + +// " print(" + +// " $tds->rename(~id,~newId)" + +// " ,2);" + +// "}\n"); +// this.execute("test():Any[*]"); +// runtime.delete("fromString.pure"); +// +// //-------------------------------------------------------------------- +// +// compileTestSource("fromString.pure", +// "function test():Any[*]\n" + +// "{ " + +// " let tds = #TDS\n" + +// " id, name\n" + +// " 1, George\n" + +// " 2, Pierre\n" + +// " 3, Sachin\n" + +// " 4, David\n" + +// " #;\n" + +// "\n" + +// " print(" + +// " $tds->concatenate($tds)->size()" + +// " ,2);" + +// "}\n"); +// this.execute("test():Any[*]"); +// runtime.delete("fromString.pure"); +// +// +// //-------------------------------------------------------------------- +// +// compileTestSource("fromString.pure", +// "function test():Any[*]\n" + +// "{ " + +// " let tds = #TDS\n" + +// " id, name\n" + +// " 1, George\n" + +// " 2, Pierre\n" + +// " 3, Sachin\n" + +// " 4, David\n" + +// " #;\n" + +// "\n" + +// " print(" + +// " $tds->filter(x | $x.id > 2)->size()" + +// " ,2);" + +// "}\n"); +// this.execute("test():Any[*]"); +// runtime.delete("fromString.pure"); +// +// //-------------------------------------------------------------------- + } +} diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/ColumnValue.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/ColumnValue.java new file mode 100644 index 00000000000..030cda38528 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/ColumnValue.java @@ -0,0 +1,84 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.shared; + +import io.deephaven.csv.parsers.DataType; +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.MutableList; + +public class ColumnValue +{ + String name; + DataType type; + Object result; + boolean[] nulls = null; + + public ColumnValue(String name, DataType type, Object result) + { + this.name = name; + this.type = type; + this.result = result; + } + + public ColumnValue(String name, DataType type, Object result, boolean[] nulls) + { + this.name = name; + this.type = type; + this.result = result; + this.nulls = nulls; + } + + @Override + public String toString() + { + MutableList res; + if (type == DataType.INT) + { + int[] val = (int[]) result; + res = extracted(i -> String.valueOf(val[i]), val.length); + } + else if (type == DataType.STRING) + { + String[] val = (String[]) result; + res = extracted(i -> val[i], val.length); + } + else if (type == DataType.DOUBLE) + { + double[] val = (double[]) result; + res = extracted(i -> String.valueOf(val[i]), val.length); + } + else + { + throw new RuntimeException("TODO"); + } + + return "ColumnValue{" + + "name='" + name + '\'' + + ", type=" + type + + ", result=" + res.makeString("[", ",", "]") + + '}'; + } + + private MutableList extracted(Function function, int size) + { + MutableList res = Lists.mutable.empty(); + for (int i = 0; i < size; i++) + { + res.add(function.valueOf(i)); + } + return res; + } +} diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java index 78b8ea70b0c..c777e489c98 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java @@ -56,6 +56,47 @@ public TestTDS newTDS() return new TestTDS(); } + public TestTDS newEmptyTDS() + { + return newTDS(columnsOrdered, columnType, 0); + } + + public TestTDS newNullTDS() + { + TestTDS testTDS = newTDS(columnsOrdered, columnType, 1); + for (String col : columnsOrdered) + { + testTDS.isNullByColumn.put(col, new boolean[]{true}); + switch (columnType.get(col)) + { + case INT: + { + testTDS.dataByColumnName.put(col, new int[(int) testTDS.rowCount]); + break; + } + case CHAR: + { + testTDS.dataByColumnName.put(col, new char[(int) this.rowCount]); + break; + } + case STRING: + { + testTDS.dataByColumnName.put(col, new String[(int) this.rowCount]); + break; + } + case FLOAT: + case DOUBLE: + { + testTDS.dataByColumnName.put(col, new double[(int) this.rowCount]); + break; + } + default: + throw new RuntimeException("ERROR " + columnType.get(col) + " not supported yet!"); + } + } + return testTDS; + } + public TestTDS newTDS(MutableList columnOrdered, MutableMap columnType, int rows) { return new TestTDS(columnOrdered, columnType, rows); @@ -489,7 +530,31 @@ private boolean[] concatenate(boolean[] set1, boolean[] set2) return _copy; } + public TestTDS addColumn(ColumnValue columnValue) + { + if (columnValue.nulls == null) + { + return addColumn(columnValue.name, columnValue.type, columnValue.result); + } + else + { + return addColumn(columnValue.name, columnValue.type, columnValue.result, columnValue.nulls); + } + } + + public Pair>> wrapFullTDS() + { + return Tuples.pair(this.copy(), org.eclipse.collections.impl.factory.Lists.mutable.with(Tuples.pair(0, (int) this.getRowCount()))); + } + public TestTDS addColumn(String name, DataType dataType, Object res) + { + boolean[] array = new boolean[Array.getLength(res)]; + Arrays.fill(array, Boolean.FALSE); + return addColumn(name, dataType, res, array); + } + + public TestTDS addColumn(String name, DataType dataType, Object res, Object nulls) { int size = Array.getLength(res); if (this.rowCount == 0) @@ -509,9 +574,7 @@ public TestTDS addColumn(String name, DataType dataType, Object res) case CHAR: case FLOAT: case DOUBLE: - boolean[] array = new boolean[(int) this.rowCount]; - Arrays.fill(array, Boolean.FALSE); - isNullByColumn.put(name, array); + isNullByColumn.put(name, nulls); } return this; } @@ -1014,4 +1077,19 @@ public MutableList getColumnNames() { return this.columnsOrdered; } + + public static Pair>> sortPartitions(MutableList transformedSort, Pair>> source) + { + if (!transformedSort.isEmpty()) + { + TestTDS res = source.getOne().newEmptyTDS(); + for (int j = 0; j < source.getTwo().size(); j++) + { + Pair r = source.getTwo().get(j); + res = res.concatenate(source.getOne().slice(r.getOne(), r.getTwo()).sort(transformedSort).getOne()); + } + source = Tuples.pair(res, source.getTwo()); + } + return source; + } } diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/src/main/java/org/finos/legend/engine/ide/PureIDELight.java b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/src/main/java/org/finos/legend/engine/ide/PureIDELight.java index 1b1d18d3a1a..953f38e9cd3 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/src/main/java/org/finos/legend/engine/ide/PureIDELight.java +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/src/main/java/org/finos/legend/engine/ide/PureIDELight.java @@ -37,18 +37,30 @@ public static void main(String[] args) throws Exception protected MutableList buildRepositories(SourceLocationConfiguration sourceLocationConfiguration) { return Lists.mutable.empty() + // --- MINIMUM ----- .with(this.buildCore("legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-metadata-pure", "ide_metadata")) - .with(this.buildCore("legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-standard/legend-engine-pure-functions-standard-pure", "functions_standard")) + .with(this.build("legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-pure", "pure_ide", false)) + .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure", "relational")) + .with(this.buildCore("legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-pure", "servicestore")) + .with(this.buildCore("legend-engine-xts-service/legend-engine-language-pure-dsl-service-pure", "service")) + .with(this.buildCore("legend-engine-xts-json/legend-engine-xt-json-pure", "external_format_json")) + .with(this.buildCore("legend-engine-xts-generation/legend-engine-language-pure-dsl-generation-pure", "generation")) + .with(this.buildCore("legend-engine-xts-authentication/legend-engine-xt-authentication-pure", "authentication")) + .with(this.buildCore("legend-engine-xts-data-space/legend-engine-xt-data-space-pure-metamodel", "data_space_metamodel")) + .with(this.buildCore("legend-engine-xts-diagram/legend-engine-xt-diagram-pure-metamodel", "diagram-metamodel")) .with(this.buildCore("legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-unclassified/legend-engine-pure-functions-unclassified-pure", "functions_unclassified")) + .with(this.buildCore("legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-json/legend-engine-pure-functions-json-pure", "functions_json")) .with(this.buildCore("legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure", "functions_relation")) + .with(this.buildCore("legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-standard/legend-engine-pure-functions-standard-pure", "functions_standard")) .with(this.buildCore("legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core", "")) + // --- MINIMUM ----- + .with(this.buildCore("legend-engine-xts-protocol-java-generation/legend-engine-protocol-generation-pure", "protocol_generation")) .with(this.buildCore("legend-engine-xts-persistence/legend-engine-xt-persistence-pure", "persistence")) .with(this.buildCore("legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure", "function_activator")) .with(this.buildCore("legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure", "snowflakeapp")) .with(this.buildCore("legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure", "bigqueryfunction")) .with(this.buildCore("legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure", "hostedservice")) - .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure", "relational")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-javaPlatformBinding-pure", "relational-java-platform-binding")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-pure", "relational_sqlserver")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure", "relational_duckdb")) @@ -66,12 +78,10 @@ protected MutableList buildRepositories(SourceLocationCon .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/legend-engine-xt-relationalStore-sybaseiq-pure", "relational_sybaseiq")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure", "relational_sparksql")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-pure", "relational_store_entitlement")) - .with(this.buildCore("legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-pure", "servicestore")) .with(this.buildCore("legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-javaPlatformBinding-pure", "servicestore-java-platform-binding")) .with(this.buildCore("legend-engine-xts-text/legend-engine-xt-text-pure-metamodel", "text-metamodel")) .with(this.buildCore("legend-engine-xts-data-space/legend-engine-xt-data-space-pure-metamodel", "data-space-metamodel")) .with(this.buildCore("legend-engine-xts-data-space/legend-engine-xt-data-space-pure", "data-space")) - .with(this.buildCore("legend-engine-xts-diagram/legend-engine-xt-diagram-pure-metamodel", "diagram-metamodel")) .with(this.buildCore("legend-engine-xts-diagram/legend-engine-xt-diagram-pure", "diagram")) .with(this.buildCore("legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure", "external-format-flatdata")) .with(this.buildCore("legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-pure", "external-format-flatdata-java-platform-binding")) @@ -100,13 +110,11 @@ protected MutableList buildRepositories(SourceLocationCon .with(this.buildCore("legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure", "java-platform-binding")) .with(this.buildCore("legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel", "external-query-sql-metamodel")) .with(this.buildCore("legend-engine-xts-sql/legend-engine-xt-sql-pure", "external-query-sql")) - .with(this.buildCore("legend-engine-xts-authentication/legend-engine-xt-authentication-pure", "authentication")) .with(this.buildCore("legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-pure-specification-metamodel", "elasticsearch_specification_metamodel")) .with(this.buildCore("legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test", "elasticsearch_execution_test")) .with(this.buildCore("legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel", "elasticsearch_seven_metamodel")) .with(this.buildCore("legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-pure", "nonrelational-mongodb")) .with(this.buildCore("legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-javaPlatformBinding-pure", "nonrelational-mongodb-java-platform-binding")) - .with(this.buildCore("legend-engine-xts-service/legend-engine-language-pure-dsl-service-pure", "service")) .with(this.buildCore("legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure", "external-tableformat-iceberg")) .with(this.buildCore("legend-engine-xts-arrow/legend-engine-xt-arrow-pure", "external-format-arrow")) .with(this.buildCore("legend-engine-xts-relationalai/legend-engine-xt-relationalai-pure", "external-query-relationalai")) @@ -123,12 +131,18 @@ protected void postInit() } protected MutableFSCodeStorage buildCore(String path, String module) + { + return build(path, module, true); + } + + protected MutableFSCodeStorage build(String path, String module, boolean core) { String resourceDir = path + "/src/main/resources/"; - String moduleName = "".equals(module) ? "core" : ("core_" + module.replace("-", "_")); + String moduleName = core ? "".equals(module) ? "core" : ("core_" + module.replace("-", "_")) : module.replace("-", "_"); return new MutableFSCodeStorage( GenericCodeRepository.build(Paths.get(resourceDir + moduleName + ".definition.json")), Paths.get(resourceDir + moduleName) ); } + } diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-pure/pom.xml b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-pure/pom.xml index 9cb65b94137..aa35ce1b729 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-pure/pom.xml +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-pure/pom.xml @@ -141,17 +141,6 @@ legend-pure-m2-dsl-tds-grammar runtime - - org.finos.legend.engine - legend-engine-pure-functions-relation-pure - runtime - - - org.finos.legend.engine - legend-engine-pure-runtime-java-extension-compiled-functions-relation - ${project.version} - runtime - org.finos.legend.engine diff --git a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java index d3b8f6bdfe1..3c0d9a9f430 100644 --- a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java +++ b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java @@ -43,6 +43,17 @@ public class Test_JAVA_RelationFunction_PCT extends PCTReportConfiguration // Extend pack("meta::pure::functions::relation::tests::extend", "\"meta::pure::functions::relation::extend_Relation_1__FuncColSpec_1__Relation_1_ is not supported yet!\""), one("meta::pure::functions::relation::tests::extend::testSimpleMultipleColumns_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1__FuncColSpecArray_1__Relation_1_ is not supported yet!\""), + // OLAP + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowMultipleColums_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1__AggColSpecArray_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindow_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1__AggColSpec_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1___Window_1__AggColSpecArray_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowUsingLag_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1___Window_1__AggColSpec_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1___Window_1__AggColSpec_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindowMultipleColumns_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1___Window_1__AggColSpecArray_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindow_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1___Window_1__AggColSpec_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithMultiplePartitionsAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1___Window_1__FuncColSpecArray_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1___Window_1__FuncColSpecArray_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"meta::pure::functions::relation::extend_Relation_1___Window_1__FuncColSpec_1__Relation_1_ is not supported yet!\""), // Filter pack("meta::pure::functions::relation::tests::filter", "\"meta::pure::functions::relation::filter_Relation_1__Function_1__Relation_1_ is not supported yet!\""), @@ -84,7 +95,8 @@ public class Test_JAVA_RelationFunction_PCT extends PCTReportConfiguration // BUG: Column name with special characters is not properly escaped one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "\"meta::pure::functions::relation::select_Relation_1__ColSpec_1__Relation_1_ is not supported yet!\"") - ); + + ); public static Test suite() { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java index 0e9a5900df2..4c6dc686a6c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java @@ -35,7 +35,20 @@ public class Test_Relational_DuckDB_RelationFunctions_PCT extends PCTReportConfi private static final ReportScope reportScope = RelationCodeRepositoryProvider.relationFunctions; private static final Adapter adapter = CoreExternalTestConnectionCodeRepositoryProvider.duckDBAdapter; private static final String platform = "compiled"; - private static final MutableList expectedFailures = Lists.mutable.empty(); + private static final MutableList expectedFailures = Lists.mutable.with( + // Extend - OLAP + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowChainedWithSimple_Function_1__Boolean_1_", "\"Match failure: PlatformClusteredValueSpecificationObject instanceOf PlatformClusteredValueSpecification\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowMultipleColums_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'extend_Relation_1__AggColSpecArray_1__Relation_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindow_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'extend_Relation_1__AggColSpec_1__Relation_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowUsingLag_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithMultiplePartitionsAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\"") + ); public static Test suite() { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java index 417f703b3eb..82aaf686104 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java @@ -36,8 +36,21 @@ public class Test_Relational_H2_RelationFunctions_PCT extends PCTReportConfigura private static final Adapter adapter = CoreExternalTestConnectionCodeRepositoryProvider.H2Adapter; private static final String platform = "compiled"; private static final MutableList expectedFailures = Lists.mutable.with( + // Extend - OLAP + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowChainedWithSimple_Function_1__Boolean_1_", "\"Match failure: PlatformClusteredValueSpecificationObject instanceOf PlatformClusteredValueSpecification\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowMultipleColums_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'extend_Relation_1__AggColSpecArray_1__Relation_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindow_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'extend_Relation_1__AggColSpec_1__Relation_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowUsingLag_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithMultiplePartitionsAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "\"Unexpected error executing function with params [Anonymous_Lambda]\"") - ); + ); public static Test suite() { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java index 5e3a4da6ac9..2eca53df7c0 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java @@ -30,14 +30,28 @@ import static org.finos.legend.engine.test.shared.framework.PureTestHelperFramework.wrapSuite; + public class Test_Relational_Postgres_RelationFunctions_PCT extends PCTReportConfiguration { private static final ReportScope reportScope = RelationCodeRepositoryProvider.relationFunctions; private static final Adapter adapter = CoreExternalTestConnectionCodeRepositoryProvider.postgresAdapter; private static final String platform = "compiled"; private static final MutableList expectedFailures = Lists.mutable.with( + // Extend - OLAP + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowChainedWithSimple_Function_1__Boolean_1_", "\"Match failure: PlatformClusteredValueSpecificationObject instanceOf PlatformClusteredValueSpecification\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowMultipleColums_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'extend_Relation_1__AggColSpecArray_1__Relation_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindow_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'extend_Relation_1__AggColSpec_1__Relation_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowUsingLag_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithMultiplePartitionsAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "\"Unexpected error executing function with params [Anonymous_Lambda]\"") - ); + ); public static Test suite() { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/snowflake/pct/Test_Relational_Snowflake_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/snowflake/pct/Test_Relational_Snowflake_RelationFunctions_PCT.java index 624dc6a4efa..ca1f484aada 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/snowflake/pct/Test_Relational_Snowflake_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/snowflake/pct/Test_Relational_Snowflake_RelationFunctions_PCT.java @@ -36,8 +36,21 @@ public class Test_Relational_Snowflake_RelationFunctions_PCT extends PCTReportCo private static final Adapter adapter = CoreExternalTestConnectionCodeRepositoryProvider.snowflakeAdapter; private static final String platform = "compiled"; private static final MutableList expectedFailures = Lists.mutable.with( + // Extend - OLAP + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowChainedWithSimple_Function_1__Boolean_1_", "\"Match failure: PlatformClusteredValueSpecificationObject instanceOf PlatformClusteredValueSpecification\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowMultipleColums_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'extend_Relation_1__AggColSpecArray_1__Relation_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggNoWindow_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'extend_Relation_1__AggColSpec_1__Relation_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowUsingLag_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithMultiplePartitionsAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindowMultipleColumns_Function_1__Boolean_1_", "\"The system is trying to get an element at offset 1 where the collection is of size 1\""), + one("meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindow_Function_1__Boolean_1_", "\"Cast exception: StoreRoutingStrategy cannot be cast to StoreMappingRoutingStrategy\""), + one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "\"Unexpected error executing function with params [Anonymous_Lambda]\"") - ); + ); public static Test suite() { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java index 8ae5f91fc90..a48e9efadc2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java @@ -119,6 +119,7 @@ import org.finos.legend.pure.m3.coreinstance.meta.pure.mapping.Mapping; import org.finos.legend.pure.m3.coreinstance.meta.pure.mapping.SetImplementation; import org.finos.legend.pure.m3.coreinstance.meta.pure.mapping.aggregationAware.AggregationAwareSetImplementation; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.Multiplicity; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification; @@ -177,49 +178,49 @@ public Iterable> getExtraProcessors() { return Lists.immutable.with( Processor.newProcessor( - Database.class, - (Database srcDatabase, CompileContext context) -> - { - org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = new Root_meta_relational_metamodel_Database_Impl(srcDatabase.name, SourceInformationHelper.toM3SourceInformation(srcDatabase.sourceInformation), null)._name(srcDatabase.name); + Database.class, + (Database srcDatabase, CompileContext context) -> + { + org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = new Root_meta_relational_metamodel_Database_Impl(srcDatabase.name, SourceInformationHelper.toM3SourceInformation(srcDatabase.sourceInformation), null)._name(srcDatabase.name); - database._classifierGenericType(new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) - ._rawType(context.pureModel.getType("meta::relational::metamodel::Database"))); - return database; - }, - (Database srcDatabase, CompileContext context) -> - { - org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = HelperRelationalBuilder.getDatabase(context.pureModel.buildPackageString(srcDatabase._package, srcDatabase.name), srcDatabase.sourceInformation, context); - if (!srcDatabase.includedStores.isEmpty()) - { - database._includes(ListIterate.collect(srcDatabase.includedStores, include -> HelperRelationalBuilder.resolveDatabase(context.pureModel.addPrefixToTypeReference(include.path), include.sourceInformation, context))); - } - database._schemas(ListIterate.collect(srcDatabase.schemas, _schema -> HelperRelationalBuilder.processDatabaseSchema(_schema, context, database))); - }, - (Database srcDatabase, CompileContext context) -> - { - org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = HelperRelationalBuilder.getDatabase(context.pureModel.buildPackageString(srcDatabase._package, srcDatabase.name), srcDatabase.sourceInformation, context); - ListIterate.forEach(srcDatabase.schemas, _schema -> HelperRelationalBuilder.processDatabaseSchemaViewsFirstPass(_schema, context, database)); - }, - (Database srcDatabase, CompileContext context) -> - { - org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = HelperRelationalBuilder.getDatabase(context.pureModel.buildPackageString(srcDatabase._package, srcDatabase.name), srcDatabase.sourceInformation, context); - // TODO checkForDuplicatesByName for filters/joins - database._joins(srcDatabase.joins == null ? Lists.fixedSize.empty() : ListIterate.collect(srcDatabase.joins, join -> HelperRelationalBuilder.processDatabaseJoin(join, context, database))) - ._filters(srcDatabase.filters == null ? Lists.fixedSize.empty() : ListIterate.collect(srcDatabase.filters, filter -> HelperRelationalBuilder.processDatabaseFilter(filter, context, database))) - ._stereotypes(srcDatabase.stereotypes == null ? Lists.fixedSize.empty() : ListIterate.collect(srcDatabase.stereotypes, stereotypePointer -> context.resolveStereotype(stereotypePointer.profile, stereotypePointer.value, stereotypePointer.profileSourceInformation, stereotypePointer.sourceInformation))); - }, - (Database srcDatabase, CompileContext context) -> - { - org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = HelperRelationalBuilder.getDatabase(context.pureModel.buildPackageString(srcDatabase._package, srcDatabase.name), srcDatabase.sourceInformation, context); - ListIterate.forEach(srcDatabase.schemas, _schema -> HelperRelationalBuilder.processDatabaseSchemaViewsSecondPass(_schema, context, database)); - } + database._classifierGenericType(new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) + ._rawType(context.pureModel.getType("meta::relational::metamodel::Database"))); + return database; + }, + (Database srcDatabase, CompileContext context) -> + { + org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = HelperRelationalBuilder.getDatabase(context.pureModel.buildPackageString(srcDatabase._package, srcDatabase.name), srcDatabase.sourceInformation, context); + if (!srcDatabase.includedStores.isEmpty()) + { + database._includes(ListIterate.collect(srcDatabase.includedStores, include -> HelperRelationalBuilder.resolveDatabase(context.pureModel.addPrefixToTypeReference(include.path), include.sourceInformation, context))); + } + database._schemas(ListIterate.collect(srcDatabase.schemas, _schema -> HelperRelationalBuilder.processDatabaseSchema(_schema, context, database))); + }, + (Database srcDatabase, CompileContext context) -> + { + org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = HelperRelationalBuilder.getDatabase(context.pureModel.buildPackageString(srcDatabase._package, srcDatabase.name), srcDatabase.sourceInformation, context); + ListIterate.forEach(srcDatabase.schemas, _schema -> HelperRelationalBuilder.processDatabaseSchemaViewsFirstPass(_schema, context, database)); + }, + (Database srcDatabase, CompileContext context) -> + { + org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = HelperRelationalBuilder.getDatabase(context.pureModel.buildPackageString(srcDatabase._package, srcDatabase.name), srcDatabase.sourceInformation, context); + // TODO checkForDuplicatesByName for filters/joins + database._joins(srcDatabase.joins == null ? Lists.fixedSize.empty() : ListIterate.collect(srcDatabase.joins, join -> HelperRelationalBuilder.processDatabaseJoin(join, context, database))) + ._filters(srcDatabase.filters == null ? Lists.fixedSize.empty() : ListIterate.collect(srcDatabase.filters, filter -> HelperRelationalBuilder.processDatabaseFilter(filter, context, database))) + ._stereotypes(srcDatabase.stereotypes == null ? Lists.fixedSize.empty() : ListIterate.collect(srcDatabase.stereotypes, stereotypePointer -> context.resolveStereotype(stereotypePointer.profile, stereotypePointer.value, stereotypePointer.profileSourceInformation, stereotypePointer.sourceInformation))); + }, + (Database srcDatabase, CompileContext context) -> + { + org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database database = HelperRelationalBuilder.getDatabase(context.pureModel.buildPackageString(srcDatabase._package, srcDatabase.name), srcDatabase.sourceInformation, context); + ListIterate.forEach(srcDatabase.schemas, _schema -> HelperRelationalBuilder.processDatabaseSchemaViewsSecondPass(_schema, context, database)); + } ), Processor.newProcessor( RelationalMapper.class, Lists.fixedSize.with(PackageableRuntime.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.Store.class), (relationalMapper, context) -> { - Root_meta_relational_metamodel_RelationalMapper metamodel = new Root_meta_relational_metamodel_RelationalMapper_Impl(relationalMapper.name, null, context.pureModel.getClass("meta::relational::metamodel::RelationalMapper"))._name(relationalMapper.name); + Root_meta_relational_metamodel_RelationalMapper metamodel = new Root_meta_relational_metamodel_RelationalMapper_Impl(relationalMapper.name, null, context.pureModel.getClass("meta::relational::metamodel::RelationalMapper"))._name(relationalMapper.name); relationalMappersIndex.put(context.pureModel.buildPackageString(relationalMapper._package, relationalMapper.name), metamodel); return metamodel; }, @@ -772,7 +773,7 @@ public List c.getName().equals(schemaName)).getFirst(); if (schema == null) { - throw new EngineException(schemaName == null ? "The database " + store._name() + " has no schemas" : "The schema " + schemaName + " can't be found in the store " + store._name(), accessor.sourceInformation, EngineErrorType.COMPILATION); + throw new EngineException(schemaName == null ? "The database " + store._name() + " has no schemas" : "The schema " + schemaName + " can't be found in the store " + store._name(), accessor.sourceInformation, EngineErrorType.COMPILATION); } Table table = schema._tables().select(c -> c.getName().equals(tableName)).getFirst(); if (table == null) @@ -792,7 +793,7 @@ public List - 5.10.0 + 5.11.0 0.25.4