Skip to content

Commit

Permalink
Register handlers for missing TDS extension functions: zScore, iqrCla…
Browse files Browse the repository at this point in the history 
…ssify (#2453)
  • Loading branch information
@aormerod-gs
aormerod-gs authored Nov 13, 2023
1 parent f32663e commit e6138a5
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 2 deletions.
2 changes: 0 additions & 2 deletions ...in/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2078,8 +2078,6 @@ private Map<String, Dispatch> buildDispatch()
map.put("meta::pure::tds::distinct_TabularDataSet_1__TabularDataSet_1_", (List<ValueSpecification> ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()));
map.put("meta::pure::tds::drop_TabularDataSet_1__Integer_1__TabularDataSet_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::tds::extend_TabularDataSet_1__BasicColumnSpecification_MANY__TabularDataSet_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "BasicColumnSpecification".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::tds::extensions::columnValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", (List<ValueSpecification> ps) -> ps.size() == 5 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(1)._genericType()._rawType()._name()) && matchOneMany(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "String".equals(ps.get(2)._genericType()._rawType()._name())) && matchOneMany(ps.get(3)._multiplicity()) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || "String".equals(ps.get(3)._genericType()._rawType()._name())) && matchOneMany(ps.get(4)._multiplicity()) && ("Nil".equals(ps.get(4)._genericType()._rawType()._name()) || "String".equals(ps.get(4)._genericType()._rawType()._name())));
map.put("meta::pure::tds::extensions::columnValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", (List<ValueSpecification> ps) -> ps.size() == 4 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(1)._genericType()._rawType()._name()) && matchOneMany(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "String".equals(ps.get(2)._genericType()._rawType()._name())) && matchOneMany(ps.get(3)._multiplicity()) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || "String".equals(ps.get(3)._genericType()._rawType()._name())));
map.put("meta::pure::tds::filter_TabularDataSet_1__Function_1__TabularDataSet_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> isOne(ft._returnMultiplicity()) && ("Nil".equals(ft._returnType()._rawType()._name()) || "Boolean".equals(ft._returnType()._rawType()._name())) && check(ft._parameters().toList(), (List<? extends VariableExpression> nps) -> nps.size() == 1 && isOne(nps.get(0)._multiplicity()) && Sets.immutable.with("TDSRow", "Any").contains(nps.get(0)._genericType()._rawType()._name())))));
map.put("meta::pure::tds::groupByWithWindowSubset_K_MANY__Function_MANY__AggregateValue_MANY__String_MANY__String_MANY__String_MANY__TabularDataSet_1_", (List<ValueSpecification> ps) -> ps.size() == 6 && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> check(ft._parameters().toList(), (List<? extends VariableExpression> nps) -> nps.size() == 1 && isOne(nps.get(0)._multiplicity())))) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "AggregateValue".equals(ps.get(2)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || "String".equals(ps.get(3)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(4)._genericType()._rawType()._name()) || "String".equals(ps.get(4)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(5)._genericType()._rawType()._name()) || "String".equals(ps.get(5)._genericType()._rawType()._name())));
map.put("meta::pure::tds::groupBy_K_MANY__Function_MANY__AggregateValue_MANY__String_MANY__TabularDataSet_1_", (List<ValueSpecification> ps) -> ps.size() == 4 && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> check(ft._parameters().toList(), (List<? extends VariableExpression> nps) -> nps.size() == 1 && isOne(nps.get(0)._multiplicity())))) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "AggregateValue".equals(ps.get(2)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || "String".equals(ps.get(3)._genericType()._rawType()._name())));
Expand Down
6 changes: 6 additions & 0 deletions ...g/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,12 @@ public List<Function<Handlers, List<FunctionExpressionBuilderRegistrationInfo>>>
handlers.m(handlers.m(handlers.h("meta::pure::tds::extensions::rowValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> ps.size() == 5)),
handlers.m(handlers.h("meta::pure::tds::extensions::rowValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> ps.size() == 4)))
),
new FunctionExpressionBuilderRegistrationInfo(null,
handlers.m(handlers.h("meta::pure::tds::extensions::zScore_TabularDataSet_1__String_MANY__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res(ps.get(0)._genericType(), "one"), ps -> ps.size() == 4))
),
new FunctionExpressionBuilderRegistrationInfo(null,
handlers.m(handlers.h("meta::pure::tds::extensions::iqrClassify_TabularDataSet_1__String_MANY__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res(ps.get(0)._genericType(), "one"), ps -> ps.size() == 4))
),
new FunctionExpressionBuilderRegistrationInfo(Lists.mutable.with(3),
handlers.m(handlers.h("meta::pure::functions::asserts::assertEq_Any_1__Any_1__Function_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 3 && !handlers.typeOne(ps.get(2), "String")))
)
Expand Down
75 changes: 75 additions & 0 deletions ...va/org/finos/legend/engine/language/pure/compiler/test/TestRelationalFunctionHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,4 +94,79 @@ public void testRowValueDifferenceCompile()
" }:meta::pure::tds::TabularDataSet[1];" +
"}");
}

@Test
public void testCompile_zScore()
{
// Faulty stereotype
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" +
"function xx::myFunc() : Boolean[1] {\n" +
" ^Pair<String,Integer>(first = 'student1', second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])->meta::pure::tds::extensions::zScore([], ['score'], ['score zScore']);\n" +
" true;\n" +
"}\n");
}

@Test
public void testCompile_iqrClassify()
{
// Faulty stereotype
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" +
"function xx::myFunc() : Boolean[1] {\n" +
" ^Pair<String,Integer>(first = 'student1', second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])->meta::pure::tds::extensions::iqrClassify([], 'score', 'irq_classification');\n" +
" true;\n" +
"}\n");
}

@Test
public void testCompile_columnValueDifference1()
{
// Faulty stereotype
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" +
"function xx::myFunc() : Boolean[1] {\n" +
" ^Pair<Integer,Integer>(first = 1, second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])\n" +
" ->meta::pure::tds::extensions::columnValueDifference(\n" +
" ^Pair<Integer,Integer>(first = 1, second = 6)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')]),\n" +
" ['student'], ['score']);\n" +
" true;\n" +
"}\n");
}

@Test
public void testCompile_columnValueDifference2()
{
// Faulty stereotype
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" +
"function xx::myFunc() : Boolean[1] {\n" +
" ^Pair<Integer,Integer>(first = 1, second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])\n" +
" ->meta::pure::tds::extensions::columnValueDifference(\n" +
" ^Pair<Integer,Integer>(first = 1, second = 6)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')]),\n" +
" ['student'], ['score'], ['score diff']);\n" +
" true;\n" +
"}\n");
}

@Test
public void testCompile_extendWithDigestOnColumns1()
{
// Faulty stereotype
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" +
"function xx::myFunc() : Boolean[1] {\n" +
" ^Pair<Integer,Integer>(first = 1, second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])\n" +
" ->meta::pure::tds::extensions::extendWithDigestOnColumns('digest');\n" +
" true;\n" +
"}\n");
}

@Test
public void testCompile_extendWithDigestOnColumns2()
{
// Faulty stereotype
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" +
"function xx::myFunc() : Boolean[1] {\n" +
" ^Pair<Integer,Integer>(first = 1, second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])\n" +
" ->meta::pure::tds::extensions::extendWithDigestOnColumns(['student'], meta::pure::functions::hash::HashType.MD5, 'digest');\n" +
" true;\n" +
"}\n");
}

}

0 comments on commit e6138a5

Please sign in to comment.