diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml
index 4d6f892d87f..2ebf7a8cbd5 100644
--- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml
+++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml
@@ -126,21 +126,11 @@
org.finos.legend.engine
legend-engine-pure-platform-dsl-store-java
-
- org.finos.legend.engine
- legend-engine-pure-runtime-java-extension-compiled-functions-relation
- runtime
-
org.finos.legend.pure
legend-pure-runtime-java-engine-compiled
-
- org.finos.legend.engine
- legend-engine-pure-runtime-java-extension-compiled-functions-unclassified
- runtime
-
org.finos.legend.engine
diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationTypeHelper.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationTypeHelper.java
index 03627a6a5df..e98fdb65e18 100644
--- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationTypeHelper.java
+++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationTypeHelper.java
@@ -17,6 +17,7 @@
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.protocol.pure.v1.model.relationType.Column;
import org.finos.legend.engine.protocol.pure.v1.model.relationType.RelationType;
+import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.Multiplicity;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType;
import org.finos.legend.pure.m3.navigation.ProcessorSupport;
import org.finos.legend.pure.m3.navigation._package._Package;
@@ -42,6 +43,6 @@ public static RelationType convert(org.finos.legend.pure.m3.coreinstance.meta.pu
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType convert(RelationType src, ProcessorSupport processorSupport, SourceInformation sourceInformation)
{
- return _RelationType.build(ListIterate.collect(src.columns, c -> (CoreInstance) _Column.getColumnInstance(c.name, false, (GenericType) processorSupport.type_wrapGenericType(_Package.getByUserPath(c.type, processorSupport)), sourceInformation, processorSupport)).toList(), sourceInformation, processorSupport);
+ return _RelationType.build(ListIterate.collect(src.columns, c -> (CoreInstance) _Column.getColumnInstance(c.name, false, (GenericType) processorSupport.type_wrapGenericType(_Package.getByUserPath(c.type, processorSupport)), (Multiplicity) org.finos.legend.pure.m3.navigation.multiplicity.Multiplicity.newMultiplicity(0, 1, processorSupport), sourceInformation, processorSupport)).toList(), sourceInformation, processorSupport);
}
}
diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/ValueSpecificationBuilder.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/ValueSpecificationBuilder.java
index 85e15c6c73e..dbd3e600226 100644
--- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/ValueSpecificationBuilder.java
+++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/ValueSpecificationBuilder.java
@@ -304,7 +304,7 @@ private ValueSpecification proccessColSpecArray(ColSpecArray value)
cols.collect(c ->
{
Column theCol = ((RelationType) c._genericType()._typeArguments().getLast()._rawType())._columns().getFirst();
- return _Column.getColumnInstance(theCol._name(), false, _Column.getColumnType(theCol), null, processorSupport);
+ return _Column.getColumnInstance(theCol._name(), false, _Column.getColumnType(theCol), _Column.getColumnMultiplicity(theCol), null, processorSupport);
}),
null,
processorSupport
@@ -351,7 +351,7 @@ private ValueSpecification proccessColSpec(ColSpec colSpec)
ProcessorSupport processorSupport = context.pureModel.getExecutionSupport().getProcessorSupport();
if (colSpec.function1 == null)
{
- return wrapInstanceValue(buildColSpec(colSpec.name, colSpec.type == null ? null : context.pureModel.getGenericType(colSpec.type, colSpec.sourceInformation), context.pureModel, context.pureModel.getExecutionSupport().getProcessorSupport()), context.pureModel);
+ return wrapInstanceValue(buildColSpec(colSpec.name, colSpec.type == null ? null : context.pureModel.getGenericType(colSpec.type, colSpec.sourceInformation), (Multiplicity) org.finos.legend.pure.m3.navigation.multiplicity.Multiplicity.newMultiplicity(0, 1, processorSupport), context.pureModel, context.pureModel.getExecutionSupport().getProcessorSupport()), context.pureModel);
}
else if (colSpec.function2 == null)
{
@@ -365,7 +365,7 @@ else if (colSpec.function2 == null)
new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType"))
._rawType(
_RelationType.build(
- Lists.mutable.with(_Column.getColumnInstance(colSpec.name, false, funcReturnType(funcVS, context.pureModel), null, processorSupport)),
+ Lists.mutable.with(_Column.getColumnInstance(colSpec.name, false, funcReturnType(funcVS, context.pureModel), funcReturnMul(funcVS, context.pureModel), null, processorSupport)),
null,
processorSupport
)
@@ -405,7 +405,7 @@ else if (colSpec.function2 == null)
new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType"))
._rawType(
_RelationType.build(
- Lists.mutable.with(_Column.getColumnInstance(colSpec.name, false, funcReturnType(func2VS, context.pureModel), null, processorSupport)),
+ Lists.mutable.with(_Column.getColumnInstance(colSpec.name, false, funcReturnType(func2VS, context.pureModel), funcReturnMul(func2VS, context.pureModel), null, processorSupport)),
null,
processorSupport
)
@@ -712,9 +712,9 @@ public ValueSpecification processClassInstance(RootGraphFetchTree rootGraphFetch
{
org.finos.legend.pure.m3.coreinstance.meta.pure.graphFetch.GraphFetchTree tree = HelperValueSpecificationBuilder.buildGraphFetchTree(rootGraphFetchTree, this.context, null, openVariables, processingContext);
GenericType genericType = new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType"))
- ._rawType(this.context.pureModel.getType("meta::pure::graphFetch::RootGraphFetchTree"))
- ._typeArguments(FastList.newListWith(new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType"))
- ._rawType(context.resolveClass(rootGraphFetchTree._class, rootGraphFetchTree.sourceInformation))));
+ ._rawType(this.context.pureModel.getType("meta::pure::graphFetch::RootGraphFetchTree"))
+ ._typeArguments(FastList.newListWith(new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType"))
+ ._rawType(context.resolveClass(rootGraphFetchTree._class, rootGraphFetchTree.sourceInformation))));
return new Root_meta_pure_metamodel_valuespecification_InstanceValue_Impl("", SourceInformationHelper.toM3SourceInformation(rootGraphFetchTree.sourceInformation), context.pureModel.getClass("meta::pure::metamodel::valuespecification::InstanceValue"))
._genericType(genericType)
._multiplicity(this.context.pureModel.getMultiplicity("one"))
diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java
index 40f54a488b5..2153d896782 100644
--- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java
+++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java
@@ -257,7 +257,7 @@ private static TypeAndMultiplicity getTypeAndMultiplicity(MutableList relType =
_RelationType.build(
types.flatCollect(RelationTypeAccessor::_columns)
- .collect(c -> _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), null, processorSupport)),
+ .collect(c -> _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), _Column.getColumnMultiplicity(c), null, processorSupport)),
null,
processorSupport
);
@@ -297,7 +297,7 @@ public static TypeAndMultiplicity ExtendReturnInference(List
Lists.mutable
.withAll((RichIterable>) ((RelationType) ps.get(0)._genericType()._typeArguments().getFirst()._rawType())._columns())
.withAll((RichIterable>) ((RelationType) ps.get(1)._genericType()._typeArguments().getLast()._rawType())._columns())
- .collect(c -> (CoreInstance) _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), null, processorSupport)),
+ .collect(c -> (CoreInstance) _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), _Column.getColumnMultiplicity(c), null, processorSupport)),
null,
processorSupport
);
@@ -354,7 +354,7 @@ public static TypeAndMultiplicity ExtendReturnInference(List
return Lists.mutable.with(
vs,
wrapInstanceValue(buildColSpec(foundColumn, cc.pureModel, ps), cc.pureModel),
- wrapInstanceValue(buildColSpec(secondCol.name, _Column.getColumnType(foundColumn), cc.pureModel, ps), cc.pureModel)
+ wrapInstanceValue(buildColSpec(secondCol.name, _Column.getColumnType(foundColumn), _Column.getColumnMultiplicity(foundColumn), cc.pureModel, ps), cc.pureModel)
);
};
@@ -391,12 +391,12 @@ public static InstanceValue wrapInstanceValue(MutableList values,
._values(values);
}
- public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpec buildColSpec(String name, GenericType colType, PureModel pureModel, ProcessorSupport ps)
+ public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpec buildColSpec(String name, GenericType colType, Multiplicity multiplicity, PureModel pureModel, ProcessorSupport ps)
{
GenericType firstGenericType = new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, pureModel.getClass("meta::pure::metamodel::type::generics::GenericType"))
._rawType(
_RelationType.build(
- Lists.mutable.with(_Column.getColumnInstance(name, false, colType, null, ps)),
+ Lists.mutable.with(_Column.getColumnInstance(name, false, colType, multiplicity, null, ps)),
null,
ps
)
@@ -413,16 +413,16 @@ public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpec buildColSpec(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.Column col, PureModel pureModel, ProcessorSupport ps)
{
- return buildColSpec(col._name(), _Column.getColumnType(col), pureModel, ps);
+ return buildColSpec(col._name(), _Column.getColumnType(col), _Column.getColumnMultiplicity(col), pureModel, ps);
}
public static final ParametersInference LambdaColCollectionInference = (parameters, ov, cc, pc) ->
{
ValueSpecification firstProcessedParameter = parameters.get(0).accept(new ValueSpecificationBuilder(cc, ov, pc));
GenericType gt = firstProcessedParameter._genericType();
- if (parameters.get(1) instanceof ClassInstance)
+ if (parameters.get(1) instanceof ClassInstance)
{
- ((ColSpecArray)((ClassInstance) parameters.get(1)).value).colSpecs.forEach(col -> updateSimpleLambda(col.function1, gt, new org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity(1, 1)));
+ ((ColSpecArray) ((ClassInstance) parameters.get(1)).value).colSpecs.forEach(col -> updateSimpleLambda(col.function1, gt, new org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity(1, 1)));
}
else
{
@@ -1108,11 +1108,11 @@ private void registerTDS()
{
if (c._name().equals(firstCol._name()))
{
- return (CoreInstance) _Column.getColumnInstance(secondCol._name(), false, _Column.getColumnType(c), null, processorSupport);
+ return (CoreInstance) _Column.getColumnInstance(secondCol._name(), false, _Column.getColumnType(c), _Column.getColumnMultiplicity(c), null, processorSupport);
}
else
{
- return (CoreInstance) _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), null, processorSupport);
+ return (CoreInstance) _Column.getColumnInstance(c._name(), false, _Column.getColumnType(c), _Column.getColumnMultiplicity(c), null, processorSupport);
}
}).toList(),
null,
@@ -1897,6 +1897,11 @@ public static GenericType funcReturnType(ValueSpecification vs, PureModel pm)
return funcType(vs._genericType(), pm)._returnType();
}
+ public static Multiplicity funcReturnMul(ValueSpecification vs, PureModel pm)
+ {
+ return funcType(vs._genericType(), pm)._returnMultiplicity();
+ }
+
private GenericType funcReturnType(ValueSpecification vs)
{
return funcType(vs._genericType(), this.pureModel)._returnType();
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure
index 8d562764d87..ba6413389bf 100644
--- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure
@@ -256,7 +256,16 @@ function meta::pure::metamodel::serialization::grammar::printGenericType(generic
{
if ($genericType->instanceOf(GenericTypeOperation),
|let go = $genericType->cast(@GenericTypeOperation);
- $go.left->printGenericType()+if($go.type == GenericTypeOperationType.Union,|'+',|'-')+$go.right->printGenericType();,
+ if($go.left->isEmpty(),|'',|$go.left->toOne()->printGenericType()) +
+ if(
+ [
+ pair(|$go.type == GenericTypeOperationType.Union,|'+'),
+ pair(|$go.type == GenericTypeOperationType.Difference,|'-'),
+ pair(|$go.type == GenericTypeOperationType.Subset,|'⊆')
+ ],
+ |'='
+ ) +
+ $go.right->printGenericType();,
|if ($genericType.rawType->isEmpty(),
|if ($genericType.typeParameter->isEmpty(),
|'',
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/extend.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/extend.pure
index 20408134e6c..0a6a58500bd 100644
--- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/extend.pure
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/extend.pure
@@ -16,9 +16,17 @@ import meta::pure::test::pct::*;
import meta::pure::metamodel::relation::*;
native function <> meta::pure::functions::relation::extend(r:Relation[1], f:FuncColSpec<{T[1]->Any[0..1]},Z>[1]):Relation[1];
-
native function <> meta::pure::functions::relation::extend(r:Relation[1], fs:FuncColSpecArray<{T[1]->Any[*]},Z>[1]):Relation[1];
+native function <> meta::pure::functions::relation::extend(r:Relation[1], agg:AggColSpec<{T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1];
+native function <> meta::pure::functions::relation::extend(r:Relation[1], agg:AggColSpecArray<{T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1];
+
+native function <> meta::pure::functions::relation::extend(r:Relation[1], window:_Window[1], f:FuncColSpec<{Relation[1],T[1]->Any[0..1]},R>[1]):Relation[1];
+native function <> meta::pure::functions::relation::extend(r:Relation[1], window:_Window[1], f:FuncColSpecArray<{Relation[1],T[1]->Any[*]},R>[1]):Relation[1];
+
+native function <> meta::pure::functions::relation::extend(r:Relation[1], window:_Window[1], agg:AggColSpec<{Relation[1],T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1];
+native function <> meta::pure::functions::relation::extend(r:Relation[1], window:_Window[1], agg:AggColSpecArray<{Relation[1],T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1];
+
function <> meta::pure::functions::relation::tests::extend::testSimpleExtendStrShared(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
{
@@ -119,3 +127,403 @@ function <> meta::pure::functions::relation::tests::extend::testSimple
' 6,weq,7,weq_ext\n'+
'#', $res->toString());
}
+
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPAggNoWindow(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(~newCol:c|$c.id:y|$y->plus())
+ };
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol\n'+
+ ' 1,2,A,55\n'+
+ ' 2,1,B,55\n'+
+ ' 3,3,C,55\n'+
+ ' 4,4,D,55\n'+
+ ' 5,2,E,55\n'+
+ ' 6,1,F,55\n'+
+ ' 7,3,G,55\n'+
+ ' 8,1,H,55\n'+
+ ' 9,5,I,55\n'+
+ ' 10,0,J,55\n'+
+ '#', $res->sort(~id->ascending())->toString());
+}
+
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowMultipleColums(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(~[newCol:c|$c.id:y|$y->plus(), other:c|$c.id:y|$y->plus() +1])
+ };
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol,other\n'+
+ ' 1,2,A,55,56\n'+
+ ' 2,1,B,55,56\n'+
+ ' 3,3,C,55,56\n'+
+ ' 4,4,D,55,56\n'+
+ ' 5,2,E,55,56\n'+
+ ' 6,1,F,55,56\n'+
+ ' 7,3,G,55,56\n'+
+ ' 8,1,H,55,56\n'+
+ ' 9,5,I,55,56\n'+
+ ' 10,0,J,55,56\n'+
+ '#', $res->sort(~id->ascending())->toString());
+}
+
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindow(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(over(~grp), ~newCol:{w,c|$c.id}:y|$y->plus())
+ };
+
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol\n'+
+ ' 10,0,J,10\n'+
+ ' 2,1,B,16\n'+
+ ' 6,1,F,16\n'+
+ ' 8,1,H,16\n'+
+ ' 1,2,A,6\n'+
+ ' 5,2,E,6\n'+
+ ' 3,3,C,10\n'+
+ ' 7,3,G,10\n'+
+ ' 4,4,D,4\n'+
+ ' 9,5,I,9\n'+
+ '#', $res->sort(~grp->ascending())->toString());
+}
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionWindowMultipleColumns(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(over(~grp), ~[newCol:{w,c|$c.id}:y|$y->plus(), other:{w,c|$c.id}:y|$y->plus()+1])
+ };
+
+ let res = $f->eval($expr);
+
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol,other\n'+
+ ' 10,0,J,10,11\n'+
+ ' 2,1,B,16,17\n'+
+ ' 6,1,F,16,17\n'+
+ ' 8,1,H,16,17\n'+
+ ' 1,2,A,6,7\n'+
+ ' 5,2,E,6,7\n'+
+ ' 3,3,C,10,11\n'+
+ ' 7,3,G,10,11\n'+
+ ' 4,4,D,4,5\n'+
+ ' 9,5,I,9,10\n'+
+ '#', $res->sort(~grp->ascending())->toString());
+}
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindow(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(over(~grp, ~id->descending()), ~newCol:{w,c|$c.name}:y|$y->joinStrings(''))
+ };
+
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol\n'+
+ ' 10,0,J,J\n'+
+ ' 8,1,H,HFB\n'+
+ ' 6,1,F,HFB\n'+
+ ' 2,1,B,HFB\n'+
+ ' 5,2,E,EA\n'+
+ ' 1,2,A,EA\n'+
+ ' 7,3,G,GC\n'+
+ ' 3,3,C,GC\n'+
+ ' 4,4,D,D\n'+
+ ' 9,5,I,I\n'+
+ '#', $res->sort(~grp->ascending())->toString());
+}
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowUsingLag(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(over(~grp, ~id->descending()), ~newCol:{w,c|$w->lag($c).name}:y|$y->joinStrings(''))
+ };
+
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol\n'+
+ ' 10,0,J,\n'+
+ ' 8,1,H,HF\n'+
+ ' 6,1,F,HF\n'+
+ ' 2,1,B,HF\n'+
+ ' 5,2,E,E\n'+
+ ' 1,2,A,E\n'+
+ ' 7,3,G,G\n'+
+ ' 3,3,C,G\n'+
+ ' 4,4,D,\n'+
+ ' 9,5,I,\n'+
+ '#', $res->sort(~grp->ascending())->toString());
+}
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPAggWithPartitionAndOrderWindowMultipleColumns(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(over(~grp, ~id->descending()), ~[newCol:{w,c|$c.name}:y|$y->joinStrings(''),other:{w,x|$x.id}:y|$y->plus()])
+ };
+
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol,other\n'+
+ ' 10,0,J,J,10\n'+
+ ' 8,1,H,HFB,16\n'+
+ ' 6,1,F,HFB,16\n'+
+ ' 2,1,B,HFB,16\n'+
+ ' 5,2,E,EA,6\n'+
+ ' 1,2,A,EA,6\n'+
+ ' 7,3,G,GC,10\n'+
+ ' 3,3,C,GC,10\n'+
+ ' 4,4,D,D,4\n'+
+ ' 9,5,I,I,9\n'+
+ '#', $res->sort(~grp->ascending())->toString());
+}
+
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindow(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(over(~grp, ~id->descending()), ~newCol:{w,c|$w->lead($c).id})
+ };
+
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol\n'+
+ ' 10,0,J,null\n'+
+ ' 8,1,H,6\n'+
+ ' 6,1,F,2\n'+
+ ' 2,1,B,null\n'+
+ ' 5,2,E,1\n'+
+ ' 1,2,A,null\n'+
+ ' 7,3,G,3\n'+
+ ' 3,3,C,null\n'+
+ ' 4,4,D,null\n'+
+ ' 9,5,I,null\n'+
+ '#', $res->sort(~grp->ascending())->toString());
+}
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPWithPartitionAndOrderWindowMultipleColumns(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(over(~grp, ~id->descending()), ~[newCol:{w,c|$w->lead($c).id}, other:{w,c|$w->first().name}])
+ };
+
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol,other\n'+
+ ' 10,0,J,null,J\n'+
+ ' 8,1,H,6,H\n'+
+ ' 6,1,F,2,H\n'+
+ ' 2,1,B,null,H\n'+
+ ' 5,2,E,1,E\n'+
+ ' 1,2,A,null,E\n'+
+ ' 7,3,G,3,G\n'+
+ ' 3,3,C,null,G\n'+
+ ' 4,4,D,null,D\n'+
+ ' 9,5,I,null,I\n'+
+ '#', $res->sort(~grp->ascending())->toString());
+}
+
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPAggNoWindowChainedWithSimple(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, name
+ 1, 2, A
+ 2, 1, B
+ 3, 3, C
+ 4, 4, D
+ 5, 2, E
+ 6, 1, F
+ 7, 3, G
+ 8, 1, H
+ 9, 5, I
+ 10, 0, J
+ #->extend(~newCol:c|$c.id:y|$y->plus())
+ ->extend(~other:x|floor($x.newCol->toOne() / $x.id->toOne()))
+ };
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,name,newCol,other\n'+
+ ' 1,2,A,55,55\n'+
+ ' 2,1,B,55,27\n'+
+ ' 3,3,C,55,18\n'+
+ ' 4,4,D,55,13\n'+
+ ' 5,2,E,55,11\n'+
+ ' 6,1,F,55,9\n'+
+ ' 7,3,G,55,7\n'+
+ ' 8,1,H,55,6\n'+
+ ' 9,5,I,55,6\n'+
+ ' 10,0,J,55,5\n'+
+ '#', $res->sort(~id->ascending())->toString());
+}
+
+function <> meta::pure::functions::relation::tests::extend::testOLAPWithMultiplePartitionsAndOrderWindowMultipleColumns(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
+{
+ let expr = {
+ | #TDS
+ id, grp, grp2, name
+ 1, 2, 2, A
+ 2, 1, 1, B
+ 3, 3, 3, C
+ 4, 4, 4, D
+ 5, 2, 2, E
+ 6, 1, 2, F
+ 7, 3, 3, G
+ 8, 1, 1, H
+ 9, 5, 5, I
+ 10, 0, 0, J
+ #->extend(over(~[grp,grp2], ~id->descending()), ~[newCol:{w,c|$w->lead($c).id}, other:{w,c|$w->first().name}])
+ };
+
+
+ let res = $f->eval($expr);
+
+ assertEquals( '#TDS\n'+
+ ' id,grp,grp2,name,newCol,other\n'+
+ ' 10,0,0,J,null,J\n'+
+ ' 8,1,1,H,2,H\n'+
+ ' 2,1,1,B,null,H\n'+
+ ' 6,1,2,F,null,F\n'+
+ ' 5,2,2,E,1,E\n'+
+ ' 1,2,2,A,null,E\n'+
+ ' 7,3,3,G,3,G\n'+
+ ' 3,3,3,C,null,G\n'+
+ ' 4,4,4,D,null,D\n'+
+ ' 9,5,5,I,null,I\n'+
+ '#', $res->sort([~grp->ascending(), ~grp2->ascending()])->toString());
+}
+
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/first.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/first.pure
new file mode 100644
index 00000000000..fdd94226ca9
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/first.pure
@@ -0,0 +1,17 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import meta::pure::metamodel::relation::*;
+
+native function meta::pure::functions::relation::first(w:Relation[1]):T[1];
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lag.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lag.pure
new file mode 100644
index 00000000000..86b29ad754d
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lag.pure
@@ -0,0 +1,26 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import meta::pure::metamodel::relation::*;
+import meta::pure::functions::relation::*;
+
+function meta::pure::functions::relation::lag(w:Relation[1],r:T[1]):T[1]
+{
+ lag($w, $r, 1);
+}
+
+function meta::pure::functions::relation::lag(w:Relation[1],r:T[1], offset:Integer[1]):T[1]
+{
+ offset($w, $r, -$offset);
+}
\ No newline at end of file
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/last.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/last.pure
new file mode 100644
index 00000000000..b08b6a436b3
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/last.pure
@@ -0,0 +1,17 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import meta::pure::metamodel::relation::*;
+
+native function meta::pure::functions::relation::last(w:Relation[1]):T[1];
\ No newline at end of file
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lead.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lead.pure
new file mode 100644
index 00000000000..94aa75af546
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/lead.pure
@@ -0,0 +1,26 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import meta::pure::metamodel::relation::*;
+import meta::pure::functions::relation::*;
+
+function meta::pure::functions::relation::lead(w:Relation[1],r:T[1]):T[1]
+{
+ lead($w, $r, 1)
+}
+
+function meta::pure::functions::relation::lead(w:Relation[1],r:T[1], offset:Integer[1]):T[1]
+{
+ offset($w, $r, $offset);
+}
\ No newline at end of file
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/offset.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/offset.pure
new file mode 100644
index 00000000000..7caaa0fd751
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/offset.pure
@@ -0,0 +1,17 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import meta::pure::metamodel::relation::*;
+
+native function meta::pure::functions::relation::offset(w:Relation[1], r:T[1], offset:Integer[1]):T[1];
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/frame.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/frame.pure
new file mode 100644
index 00000000000..1bff304f5ff
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/frame.pure
@@ -0,0 +1,19 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+Class meta::pure::functions::relation::Frame
+{
+ offsetFrom : Integer[1];
+ offsetTo : Integer[1];
+}
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/over.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/over.pure
new file mode 100644
index 00000000000..e7ae76ae112
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/over.pure
@@ -0,0 +1,79 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import meta::pure::metamodel::relation::*;
+
+Class meta::pure::functions::relation::_Window
+{
+ partition : String[*];
+ sortInfo : SortInfo[*];
+ frame : Frame[0..1];
+}
+
+function meta::pure::functions::relation::over(cols:String[*], sortInfo:SortInfo<(?:?)⊆T>[*], frame:Frame[0..1]):_Window[1]
+{
+ ^_Window
+ (
+ partition=$cols,
+ sortInfo=$sortInfo,
+ frame = $frame
+ );
+}
+
+function meta::pure::functions::relation::over(cols:ColSpec<(?:?)⊆T>[1]):_Window[1]
+{
+ ^_Window
+ (
+ partition=$cols.name
+ );
+}
+
+function meta::pure::functions::relation::over(cols:ColSpecArray<(?:?)⊆T>[1]):_Window[1]
+{
+ ^_Window
+ (
+ partition=$cols.names
+ );
+}
+
+
+function meta::pure::functions::relation::over(sortInfo:SortInfo<(?:?)⊆T>[*]):_Window[1]
+{
+ over([],$sortInfo,[])
+}
+
+function meta::pure::functions::relation::over(frame:Frame[1]):_Window[1]
+{
+ over([],[],$frame)
+}
+
+function meta::pure::functions::relation::over(cols:ColSpec<(?:?)⊆T>[1], sortInfo:SortInfo<(?:?)⊆T>[*]):_Window[1]
+{
+ over($cols.name,$sortInfo,[])
+}
+
+function meta::pure::functions::relation::over(cols:ColSpecArray<(?:?)⊆T>[1], sortInfo:SortInfo<(?:?)⊆T>[*]):_Window[1]
+{
+ over($cols.names,$sortInfo,[])
+}
+
+function meta::pure::functions::relation::over(cols:ColSpec<(?:?)⊆T>[1], frame:Frame[1]):_Window[1]
+{
+ over($cols.name,[],$frame)
+}
+
+function meta::pure::functions::relation::over(sortInfo:SortInfo<(?:?)⊆T>[*], frame:Frame[1]):_Window[1]
+{
+ over([],$sortInfo,$frame)
+}
\ No newline at end of file
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/range.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/range.pure
new file mode 100644
index 00000000000..d18f7978a85
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/range.pure
@@ -0,0 +1,22 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+Class meta::pure::functions::relation::_Range extends Frame
+{
+}
+
+function meta::pure::functions::relation::_range(offsetFrom:Integer[1], offsetTo:Integer[1]):_Range[1]
+{
+ ^_Range(offsetFrom = $offsetFrom, offsetTo = $offsetTo);
+}
\ No newline at end of file
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/rows.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/rows.pure
new file mode 100644
index 00000000000..3d37289edc2
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/olap/rows.pure
@@ -0,0 +1,22 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+Class meta::pure::functions::relation::Rows extends Frame
+{
+}
+
+function meta::pure::functions::relation::rows(offsetFrom:Integer[1], offsetTo:Integer[1]):Rows[1]
+{
+ ^Rows(offsetFrom = $offsetFrom, offsetTo = $offsetTo);
+}
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/rank.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/rank.pure
new file mode 100644
index 00000000000..410e9acc63f
--- /dev/null
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/rank.pure
@@ -0,0 +1,17 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import meta::pure::metamodel::relation::*;
+
+native function meta::pure::functions::relation::rank(w:Relation[1],r:T[1]):Integer[1];
\ No newline at end of file
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java
index 2ae6a9b6663..e06c1c6f474 100644
--- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java
@@ -51,6 +51,13 @@ public List getExtraNatives()
new Join(),
new Extend(),
new ExtendArray(),
+ new ExtendAgg(),
+ new ExtendAggArray(),
+ new ExtendWindowAgg(),
+ new ExtendWindowAggArray(),
+ new ExtendWindowFunc(),
+ new ExtendWindowFuncArray(),
+ new First(),
new Drop(),
new Sort(),
new Rename(),
@@ -60,7 +67,8 @@ public List getExtraNatives()
new Slice(),
new Distinct(),
new Select(),
- new SelectArray()
+ new SelectArray(),
+ new Offset()
);
}
diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java
index e2efb5ea2ee..900b3465afd 100644
--- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java
+++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java
@@ -18,6 +18,7 @@
import org.eclipse.collections.api.RichIterable;
import org.eclipse.collections.api.block.function.Function2;
import org.eclipse.collections.api.block.function.Function3;
+import org.eclipse.collections.api.block.procedure.Procedure2;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Sets;
import org.eclipse.collections.api.list.ListIterable;
@@ -34,9 +35,12 @@
import org.finos.legend.pure.m4.coreinstance.CoreInstance;
import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport;
import org.finos.legend.pure.runtime.java.compiled.generation.processors.support.CompiledSupport;
+import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.NullRowContainer;
import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.RowContainer;
import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.TDSContainer;
import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.TestTDSCompiled;
+import org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.shared.Win;
+import org.finos.legend.pure.runtime.java.extension.external.relation.shared.ColumnValue;
import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortDirection;
import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortInfo;
import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS;
@@ -45,6 +49,7 @@
public class RelationNativeImplementation
{
+
public static TestTDSCompiled getTDS(Object value)
{
return value instanceof TDSContainer ?
@@ -120,7 +125,7 @@ public static Relation select(Relation r, Col
public static Relation concatenate(Relation rel1, Relation rel2, ExecutionSupport es)
{
ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
- return new TDSContainer((TestTDSCompiled) RelationNativeImplementation.getTDS(rel1).concatenate((TestTDSCompiled) RelationNativeImplementation.getTDS(rel2)), ps);
+ return new TDSContainer((TestTDSCompiled) RelationNativeImplementation.getTDS(rel1).concatenate(RelationNativeImplementation.getTDS(rel2)), ps);
}
public static Relation filter(Relation rel, Function2 pureFunction, ExecutionSupport es)
@@ -138,58 +143,164 @@ public static Relation filter(Relation rel, Functi
return new TDSContainer((TestTDSCompiled) tds.drop(list), ps);
}
- public static class ColFuncSpecTrans
+ public static T offset(Relation w, T r, long offset, ExecutionSupport es)
+ {
+ int actualOffset = ((RowContainer) r).getRow() + (int) offset;
+ if (actualOffset < 0 || actualOffset >= ((TDSContainer) w).tds.getRowCount())
+ {
+ return (T) new NullRowContainer();
+ }
+ return (T) new RowContainer(RelationNativeImplementation.getTDS(w), actualOffset);
+ }
+
+ public static Object first(Relation rel, ExecutionSupport es)
+ {
+ return new RowContainer(RelationNativeImplementation.getTDS(rel), 0);
+ }
+
+ public abstract static class ColFuncSpecTrans
{
public String newColName;
- public Function2 func;
public String columnType;
- public ColFuncSpecTrans(String newColName, Function2 func, String columnType)
+ public ColFuncSpecTrans(String newColName, String columnType)
{
this.newColName = newColName;
- this.func = func;
this.columnType = columnType;
}
+
+ public abstract Object eval(Object win, Object row, ExecutionSupport es);
+ }
+
+ public static class ColFuncSpecTrans1 extends ColFuncSpecTrans
+ {
+ public Function2 func;
+
+ public ColFuncSpecTrans1(String newColName, Function2 func, String columnType)
+ {
+ super(newColName, columnType);
+ this.func = func;
+ }
+
+ @Override
+ public Object eval(Object win, Object row, ExecutionSupport es)
+ {
+ return func.value(row, es);
+ }
}
- public static Relation extend(Relation rel, MutableList colFuncSpecTrans, ExecutionSupport es)
+ public static class ColFuncSpecTrans2 extends ColFuncSpecTrans
+ {
+ public Function3 func;
+
+ public ColFuncSpecTrans2(String newColName, Function3 func, String columnType)
+ {
+ super(newColName, columnType);
+ this.func = func;
+ }
+
+ @Override
+ public Object eval(Object win, Object row, ExecutionSupport es)
+ {
+ return func.value(win, row, es);
+ }
+ }
+
+ public static Relation extend(Relation rel, MutableList colFuncSpecTrans, ExecutionSupport es)
+ {
+ ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
+ TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel);
+ return new TDSContainer((TestTDSCompiled) colFuncSpecTrans.injectInto((TestTDS) tds, (accTDS, colFuncSpec) -> accTDS.addColumn(performExtend(tds.wrapFullTDS(), colFuncSpec, es))), ps);
+ }
+
+ public static Relation extendAgg(Relation rel, MutableList aggColSpecTrans, ExecutionSupport es)
{
ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel);
- TestTDSCompiled t = colFuncSpecTrans.injectInto(tds, (a, b) -> performExtend(b, es, a, ps));
- return new TDSContainer(t, ps);
+ return new TDSContainer((TestTDSCompiled) aggregateTDS(tds.wrapFullTDS(), aggColSpecTrans, false, es).injectInto((TestTDS) tds, TestTDS::addColumn), ps);
+ }
+
+ public static Relation extendWinFunc(Relation rel, Win window, MutableList colFunc, ExecutionSupport es)
+ {
+ ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
+ TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel);
+
+ Pair>> sortRes = tds.sort(window.getPartition().collect(part -> new SortInfo(part, SortDirection.ASC)).toList());
+ final Pair>> sortedPartitions = TestTDS.sortPartitions(window.getSorts(), sortRes);
+
+ return new TDSContainer((TestTDSCompiled) colFunc.injectInto(sortedPartitions.getOne(), (a, b) -> a.addColumn(performExtend(sortedPartitions, b, es))), ps);
}
- private static TestTDSCompiled performExtend(ColFuncSpecTrans colFuncSpecTrans, ExecutionSupport es, TestTDSCompiled tds, ProcessorSupport ps)
+ public static Relation extendWinAgg(Relation rel, Win window, MutableList aggColSpecTrans, ExecutionSupport es)
{
+ ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
+ TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel);
+
+ Pair>> sortRes = tds.sort(window.getPartition().collect(part -> new SortInfo(part, SortDirection.ASC)).toList());
+ final Pair>> sortedPartitions = TestTDS.sortPartitions(window.getSorts(), sortRes);
+
+ return new TDSContainer((TestTDSCompiled) aggregateTDS(sortedPartitions, aggColSpecTrans, false, es).injectInto(sortedPartitions.getOne(), TestTDS::addColumn), ps);
+ }
+
+
+ private static ColumnValue performExtend(Pair>> tds, ColFuncSpecTrans colFuncSpecTrans, ExecutionSupport es)
+ {
+ long size = tds.getOne().getRowCount();
+ boolean[] nulls = new boolean[(int) size];
switch (colFuncSpecTrans.columnType)
{
case "String":
MutableList res = Lists.mutable.empty();
- for (int i = 0; i < tds.getRowCount(); i++)
- {
- res.add((String) colFuncSpecTrans.func.value(new RowContainer(tds, i), es));
- }
- return (TestTDSCompiled) tds.addColumn(colFuncSpecTrans.newColName, DataType.STRING, res.toArray(new String[0]));
+ extracted(tds, colFuncSpecTrans, es, (i, val) -> res.add((String) val));
+ return new ColumnValue(colFuncSpecTrans.newColName, DataType.STRING, res.toArray(new String[0]));
case "Integer":
- int[] resultInt = new int[(int) tds.getRowCount()];
- for (int i = 0; i < tds.getRowCount(); i++)
- {
- resultInt[i] = (int) (long) colFuncSpecTrans.func.value(new RowContainer(tds, i), es);
- }
- return (TestTDSCompiled) tds.addColumn(colFuncSpecTrans.newColName, DataType.INT, resultInt);
+ int[] resultInt = new int[(int) size];
+ extracted(tds, colFuncSpecTrans, es, (i, val) -> processWithNull(i, val, nulls, () -> resultInt[i] = (int) (long) val));
+ return new ColumnValue(colFuncSpecTrans.newColName, DataType.INT, resultInt, nulls);
case "Double":
case "Float":
- double[] resultDouble = new double[(int) tds.getRowCount()];
- for (int i = 0; i < tds.getRowCount(); i++)
- {
- resultDouble[i] = (double) colFuncSpecTrans.func.value(new RowContainer(tds, i), es);
- }
- return (TestTDSCompiled) tds.addColumn(colFuncSpecTrans.newColName, DataType.DOUBLE, resultDouble);
+ double[] resultDouble = new double[(int) size];
+ extracted(tds, colFuncSpecTrans, es, (i, val) -> processWithNull(i, val, nulls, () -> resultDouble[i] = (double) val));
+ return new ColumnValue(colFuncSpecTrans.newColName, DataType.DOUBLE, resultDouble, nulls);
}
throw new RuntimeException(colFuncSpecTrans.columnType + " not supported yet");
}
+ private static void processWithNull(Integer j, Object val, boolean[] nulls, Proc p)
+ {
+ {
+ if (val == null)
+ {
+ nulls[j] = true;
+ }
+ else
+ {
+ p.invoke();
+ }
+ }
+ }
+
+ private interface Proc
+ {
+ void invoke();
+ }
+
+ private static void extracted(Pair>> tds, ColFuncSpecTrans colFuncSpecTrans, ExecutionSupport es, Procedure2 func)
+ {
+ int size = tds.getTwo().size();
+ int k = 0;
+ for (int j = 0; j < size; j++)
+ {
+ Pair r = tds.getTwo().get(j);
+ TDSContainer winTDS = new TDSContainer((TestTDSCompiled) tds.getOne().slice(r.getOne(), r.getTwo()), ((CompiledExecutionSupport) es).getProcessorSupport());
+ for (int i = 0; i < r.getTwo() - r.getOne(); i++)
+ {
+ func.value(k++, colFuncSpecTrans.eval(winTDS, new RowContainer((TestTDSCompiled) tds.getOne(), i), es));
+ }
+ }
+ }
+
+
public static Relation join(Relation rel1, Relation rel2, Enum joinKind, Function3 pureFunction, ExecutionSupport es)
{
ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
@@ -222,90 +333,146 @@ public static Relation sort(Relation rel, RichIter
return new TDSContainer((TestTDSCompiled) tds1.sort(collect.collect(c -> new SortInfo(c.getTwo(), SortDirection.valueOf(c.getOne()._name()))).toList()).getOne(), ps);
}
- public static class AggColSpecTrans
+ public abstract static class AggColSpecTrans
{
public String newColName;
- public Function2 map;
public Function2 reduce;
public String reduceType;
- public AggColSpecTrans(String newColName, Function2 map, Function2 reduce, String reduceType)
+ public AggColSpecTrans(String newColName, Function2 reduce, String reduceType)
{
this.newColName = newColName;
- this.map = map;
this.reduce = reduce;
this.reduceType = reduceType;
}
+
+ public abstract Object eval(Object win, Object row, ExecutionSupport es);
}
- public static Relation groupBy(Relation rel, ColSpec cols, MutableList aggColSpecTrans, ExecutionSupport es)
+ public static class AggColSpecTrans1 extends AggColSpecTrans
+ {
+ public Function2 map;
+
+ public AggColSpecTrans1(String newColName, Function2 map, Function2 reduce, String reduceType)
+ {
+ super(newColName, reduce, reduceType);
+ this.map = map;
+ }
+
+ @Override
+ public Object eval(Object win, Object row, ExecutionSupport es)
+ {
+ return map.value(row, es);
+ }
+ }
+
+ public static class AggColSpecTrans2 extends AggColSpecTrans
+ {
+ public Function3 map;
+
+ public AggColSpecTrans2(String newColName, Function3 map, Function2 reduce, String reduceType)
+ {
+ super(newColName, reduce, reduceType);
+ this.map = map;
+ }
+
+ @Override
+ public Object eval(Object win, Object row, ExecutionSupport es)
+ {
+ return map.value(win, row, es);
+ }
+ }
+
+ public static Relation groupBy(Relation rel, ColSpec cols, MutableList aggColSpecTrans, ExecutionSupport es)
{
return groupBy(rel, Lists.mutable.with(cols._name()), aggColSpecTrans, es);
}
- public static Relation groupBy(Relation rel, ColSpecArray cols, MutableList aggColSpecTransAll, ExecutionSupport es)
+ public static Relation groupBy(Relation rel, ColSpecArray cols, MutableList aggColSpecTransAll, ExecutionSupport es)
{
return groupBy(rel, Lists.mutable.withAll(cols._names()), aggColSpecTransAll, es);
}
- private static Relation groupBy(Relation rel, MutableList cols, MutableList aggColSpecTransAll, ExecutionSupport es)
+ private static Relation groupBy(Relation rel, MutableList cols, MutableList aggColSpecTransAll, ExecutionSupport es)
{
ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel);
Pair>> sortRes = tds.sort(cols.collect(name -> new SortInfo(name, SortDirection.ASC)).toList());
- int size = sortRes.getTwo().size();
-
MutableSet columnsToRemove = tds.getColumnNames().clone().toSet();
columnsToRemove.removeAll(cols.toSet());
+ TestTDS distinctTDS = sortRes.getOne()._distinct(sortRes.getTwo()).removeColumns(columnsToRemove);
- TestTDSCompiled finalTDS = (TestTDSCompiled) sortRes.getOne()._distinct(sortRes.getTwo()).removeColumns(columnsToRemove);
+ return new TDSContainer((TestTDSCompiled) aggregateTDS(sortRes, aggColSpecTransAll, true, es).injectInto(distinctTDS, TestTDS::addColumn), ps);
+ }
+ private static MutableList aggregateTDS(Pair>> sortRes, MutableList aggColSpecTransAll, boolean compress, ExecutionSupport es)
+ {
+ int size = compress ? sortRes.getTwo().size() : (int) sortRes.getOne().getRowCount();
+ MutableList columnValues = Lists.mutable.empty();
for (AggColSpecTrans aggColSpecTrans : aggColSpecTransAll)
{
- switch ((String) aggColSpecTrans.reduceType)
+ switch (aggColSpecTrans.reduceType)
{
case "String":
String[] finalRes = new String[size];
- performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sortRes, (o, j) -> finalRes[j] = (String) o);
- finalTDS.addColumn(aggColSpecTrans.newColName, DataType.STRING, finalRes);
+ performMapReduce(aggColSpecTrans, aggColSpecTrans.reduce, es, sortRes, (o, j) -> finalRes[j] = (String) o, compress);
+ columnValues.add(new ColumnValue(aggColSpecTrans.newColName, DataType.STRING, finalRes));
break;
case "Integer":
int[] finalResInt = new int[size];
- performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sortRes, (o, j) -> finalResInt[j] = (int) (long) o);
- finalTDS.addColumn(aggColSpecTrans.newColName, DataType.INT, finalResInt);
+ performMapReduce(aggColSpecTrans, aggColSpecTrans.reduce, es, sortRes, (o, j) -> finalResInt[j] = (int) (long) o, compress);
+ columnValues.add(new ColumnValue(aggColSpecTrans.newColName, DataType.INT, finalResInt));
break;
case "Double":
case "Float":
case "Number":
double[] finalResDouble = new double[size];
- performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sortRes, (o, j) -> finalResDouble[j] = (double) o);
- finalTDS.addColumn(aggColSpecTrans.newColName, DataType.FLOAT, finalResDouble);
+ performMapReduce(aggColSpecTrans, aggColSpecTrans.reduce, es, sortRes, (o, j) -> finalResDouble[j] = (double) o, compress);
+ columnValues.add(new ColumnValue(aggColSpecTrans.newColName, DataType.FLOAT, finalResDouble));
break;
default:
throw new RuntimeException(aggColSpecTrans.reduceType + " is not supported yet!");
}
}
-
- return new TDSContainer(finalTDS, ps);
+ return columnValues;
}
- private static void performMapReduce(Function2 map, Function2 reduce, ExecutionSupport es, int size, Pair>> sortRes, Function2 val)
+ private static void performMapReduce(AggColSpecTrans map, Function2 reduce, ExecutionSupport es, Pair>> sortRes, Function2 val, boolean compress)
{
+ int cursor = 0;
+ int size = sortRes.getTwo().size();
for (int j = 0; j < size; j++)
{
Pair r = sortRes.getTwo().get(j);
MutableList