Skip to content

Commit

Permalink
Implement Write API - reference implementation (#3292)
Browse files Browse the repository at this point in the history
* Implement Write API - reference implementation

* Add missing changes

* Move comment to above write expected failure
  • Loading branch information
jake-kim1 authored Dec 11, 2024
1 parent d381e3e commit cffd9b0
Show file tree
Hide file tree
Showing 11 changed files with 183 additions and 7 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
// Copyright 2024 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import meta::pure::metamodel::relation::*;
import meta::pure::functions::relation::*;
import meta::pure::store::*;
import meta::pure::test::pct::*;

native function <<PCT.function, functionType.SideEffectFunction>> meta::pure::functions::relation::write<T>(rel:Relation<T>[1], relationElementAccessor:RelationElementAccessor<T>[1]):Integer[1];

function <<PCT.test>> meta::pure::functions::relation::testWrite<T|m>(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
{
let expr = {
|
let targetTDS = meta::pure::metamodel::relation::newTDSRelationAccessor(#TDS
val,str,other
1,aaa,a
#);
let rowsAdded = #TDS
val,str,other
1,a,a
3,ewe,b
4,qw,c
5,wwe,d
6,weq,e
#->write($targetTDS);
$targetTDS->select();
};

let res = $f->eval($expr);

assertEquals( '#TDS\n'+
' val,str,other\n'+
' 1,aaa,a\n'+
' 1,a,a\n'+
' 3,ewe,b\n'+
' 4,qw,c\n'+
' 5,wwe,d\n'+
' 6,weq,e\n'+
'#', $res->toString());
}
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ public List<Native> getExtraNatives()
new CumulativeDistribution(),
new Nth(),
new DenseRank(),
new PercentRank()
new PercentRank(),
new Write()
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ public class RelationNativeImplementation

public static TestTDSCompiled getTDS(Object value)
{
if (value instanceof TDSRelationAccessor)
{
return getTDS(((TDSRelationAccessor<?>) value)._sourceElement());
}
return value instanceof TDSContainer ?
((TDSContainer) value).tds :
new TestTDSCompiled(readCsv((((CoreInstance) value).getValueForMetaPropertyToOne("csv")).getName()), ((CoreInstance) value).getValueForMetaPropertyToOne(M3Properties.classifierGenericType));
Expand Down Expand Up @@ -551,6 +555,20 @@ private static <T> Relation<? extends Object> pivot(Relation<? extends T> rel, M
return new TDSContainer(result, ps);
}

public static <T> Long write(Relation<? extends T> rel, RelationElementAccessor<? extends T> relationElementAccessor, ExecutionSupport es)
{
ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
TestTDSCompiled sourceTds = RelationNativeImplementation.getTDS(rel);
if (!(relationElementAccessor instanceof TDSRelationAccessor))
{
throw new RuntimeException("Only source element of type meta::pure::metamodel::relation::TDSRelationAccessor is supported");
}
TestTDSCompiled targetTds = RelationNativeImplementation.getTDS(relationElementAccessor);
TestTDSCompiled resultTds = (TestTDSCompiled) targetTds.concatenate(sourceTds);
relationElementAccessor._sourceElement(new TDSContainer(resultTds, ps));
return sourceTds.getRowCount();
}

private static MutableList<ColumnValue> aggregateTDS(Window window, Pair<TestTDS, MutableList<Pair<Integer, Integer>>> sortRes, MutableList<? extends AggColSpecTrans> aggColSpecTransAll, boolean compress, ExecutionSupport es)
{
int size = compress ? sortRes.getTwo().size() : (int) sortRes.getOne().getRowCount();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright 2024 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives;

import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.Relation;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationElementAccessor;
import org.finos.legend.pure.m3.execution.ExecutionSupport;
import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNativeFunctionGeneric;

public class Write extends AbstractNativeFunctionGeneric
{
public Write()
{
super("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.write", new Class[]{Relation.class, RelationElementAccessor.class, ExecutionSupport.class}, false, true, false, "write_Relation_1__RelationElementAccessor_1__Integer_1_");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Size;
import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Nth;
import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Sort;
import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Write;
import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSWithCursorCoreInstance;
import org.finos.legend.pure.runtime.java.interpreted.ExecutionSupport;
import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted;
Expand Down Expand Up @@ -115,8 +116,8 @@ public RelationExtensionInterpreted()
Tuples.pair("ntile_Relation_1__T_1__Integer_1__Integer_1_", NTile::new),
Tuples.pair("cumulativeDistribution_Relation_1___Window_1__T_1__Float_1_", CumulativeDistribution::new),
Tuples.pair("asOfJoin_Relation_1__Relation_1__Function_1__Function_1__Relation_1_", AsOfJoin::new),
Tuples.pair("asOfJoin_Relation_1__Relation_1__Function_1__Relation_1_", AsOfJoin::new)

Tuples.pair("asOfJoin_Relation_1__Relation_1__Function_1__Relation_1_", AsOfJoin::new),
Tuples.pair("write_Relation_1__RelationElementAccessor_1__Integer_1_", Write::new)
);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Copyright 2024 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives;

import org.eclipse.collections.api.list.ListIterable;
import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.stack.MutableStack;
import org.finos.legend.pure.m3.compiler.Context;
import org.finos.legend.pure.m3.exception.PureExecutionException;
import org.finos.legend.pure.m3.navigation.Instance;
import org.finos.legend.pure.m3.navigation.ProcessorSupport;
import org.finos.legend.pure.m3.navigation.ValueSpecificationBootstrap;
import org.finos.legend.pure.m4.ModelRepository;
import org.finos.legend.pure.m4.coreinstance.CoreInstance;
import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.Shared;
import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSCoreInstance;
import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS;
import org.finos.legend.pure.runtime.java.interpreted.ExecutionSupport;
import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted;
import org.finos.legend.pure.runtime.java.interpreted.VariableContext;
import org.finos.legend.pure.runtime.java.interpreted.natives.InstantiationContext;
import org.finos.legend.pure.runtime.java.interpreted.profiler.Profiler;

import java.util.Stack;

public class Write extends Shared
{
public Write(FunctionExecutionInterpreted functionExecution, ModelRepository repository)
{
super(functionExecution, repository);
}

@Override
public CoreInstance execute(ListIterable<? extends CoreInstance> params, Stack<MutableMap<String, CoreInstance>> resolvedTypeParameters, Stack<MutableMap<String, CoreInstance>> resolvedMultiplicityParameters, VariableContext variableContext, MutableStack<CoreInstance> functionExpressionCallStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, Context context, ProcessorSupport processorSupport) throws PureExecutionException
{
CoreInstance returnGenericType = getReturnGenericType(resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionCallStack, processorSupport);
TestTDS sourceTds = getTDS(params, 0, processorSupport);
CoreInstance coreInstance = params.get(1).getValueForMetaPropertyToOne("values");
if (!Instance.instanceOf(coreInstance, "meta::pure::metamodel::relation::TDSRelationAccessor", processorSupport))
{
throw new RuntimeException("Only source element of type meta::pure::metamodel::relation::TDSRelationAccessor is supported");
}
TestTDS targetTds = getTDS(coreInstance, processorSupport);
TDSCoreInstance resultTds = new TDSCoreInstance(targetTds.concatenate(sourceTds), returnGenericType, repository, processorSupport);
Instance.setValueForProperty(coreInstance, "sourceElement", resultTds, processorSupport);
return ValueSpecificationBootstrap.wrapValueSpecification(this.repository.newIntegerCoreInstance(sourceTds.getRowCount()), true, processorSupport);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.stack.MutableStack;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType;
import org.finos.legend.pure.m3.navigation.Instance;
import org.finos.legend.pure.m3.navigation.ProcessorSupport;
import org.finos.legend.pure.m3.navigation.generictype.GenericType;
import org.finos.legend.pure.m4.ModelRepository;
Expand Down Expand Up @@ -50,6 +51,10 @@ public TestTDS getTDS(ListIterable<? extends CoreInstance> params, int position,

public TestTDS getTDS(CoreInstance value, ProcessorSupport processorSupport)
{
if (Instance.instanceOf(value, "meta::pure::metamodel::relation::TDSRelationAccessor", processorSupport))
{
return getTDS(value.getValueForMetaPropertyToOne("sourceElement"), processorSupport);
}
return value instanceof TDSCoreInstance ?
((TDSCoreInstance) value).getTDS() :
new TestTDSInterpreted(readCsv((value.getValueForMetaPropertyToOne("csv")).getName()), repository, processorSupport);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,10 @@ public class Test_JAVA_RelationFunction_PCT extends PCTReportConfiguration
one("meta::pure::functions::relation::tests::composition::test_Distinct_GroupBy_Filter_Function_1__Boolean_1_", "\"meta::pure::functions::relation::filter_Relation_1__Function_1__Relation_1_ is not supported yet!\""),
one("meta::pure::functions::relation::tests::composition::test_GroupBy_Distinct_Filter_Function_1__Boolean_1_", "\"meta::pure::functions::relation::filter_Relation_1__Function_1__Relation_1_ is not supported yet!\""),
one("meta::pure::functions::relation::tests::composition::test_GroupBy_Filter_Function_1__Boolean_1_", "\"meta::pure::functions::relation::filter_Relation_1__Function_1__Relation_1_ is not supported yet!\""),
one("meta::pure::functions::relation::tests::composition::testWindowFunctionsAfterProject_Function_1__Boolean_1_", "\"meta::pure::functions::relation::sort_Relation_1__SortInfo_MANY__Relation_1_ is not supported yet!\"")
one("meta::pure::functions::relation::tests::composition::testWindowFunctionsAfterProject_Function_1__Boolean_1_", "\"meta::pure::functions::relation::sort_Relation_1__SortInfo_MANY__Relation_1_ is not supported yet!\""),

// Write
one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"meta::pure::functions::relation::select_Relation_1__Relation_1_ is not supported yet!\"")
);

public static Test suite()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ public class Test_Relational_DuckDB_RelationFunctions_PCT extends PCTReportConfi
// BUG: unsupported compositions
one("meta::pure::functions::relation::tests::composition::test_Distinct_GroupBy_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: column newCol must appear in the GROUP BY clause or be used in an aggregate function"),
one("meta::pure::functions::relation::tests::composition::test_GroupBy_Distinct_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: Referenced table \"restrict__d#2\" not found!\nCandidate tables: \"tb"),
one("meta::pure::functions::relation::tests::composition::test_GroupBy_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: column \"newCol\" must appear in the GROUP BY clause or must be part of an aggregate function.\nEither add it to the GROUP BY list, or use \"ANY_VALUE(newCol)\" if the exact value of \"newCol\" is not important.\nLINE 2: select \"str\" as \"str\", \"newCol\" as \"newCol\" from (select \"tb")
one("meta::pure::functions::relation::tests::composition::test_GroupBy_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: column \"newCol\" must appear in the GROUP BY clause or must be part of an aggregate function.\nEither add it to the GROUP BY list, or use \"ANY_VALUE(newCol)\" if the exact value of \"newCol\" is not important.\nLINE 2: select \"str\" as \"str\", \"newCol\" as \"newCol\" from (select \"tb"),

// Write test contains multiple expressions
one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"Cannot cast a collection of size 3 to multiplicity [1]\"")
);

public static Test suite()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,10 @@ public class Test_Relational_H2_RelationFunctions_PCT extends PCTReportConfigura
one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "Error while executing: Create Table leSchema.tb"),

one("meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\""),
one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\"")
one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\""),

// Write test contains multiple expressions
one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"Cannot cast a collection of size 3 to multiplicity [1]\"")
);

public static Test suite()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,10 @@ public class Test_Relational_Postgres_RelationFunctions_PCT extends PCTReportCon

// Postgres doesn't support asOf Join (May want to compensate with an OLAP equivalent if required
one("meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\""),
one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\"")
one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\""),

// Write test contains multiple expressions
one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"Cannot cast a collection of size 3 to multiplicity [1]\"")
);

public static Test suite()
Expand Down

0 comments on commit cffd9b0

Please sign in to comment.