Skip to content

Commit

Permalink
add support for mastery resolution key filter (#2578)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sahil37
sahil37 authored Jan 29, 2024
1 parent c1ba5a7 commit 3dd8572
Show file tree
Hide file tree
Showing 9 changed files with 184 additions and 9 deletions.
1 change: 1 addition & 0 deletions ...n/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryLexerGrammar.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ RESOLUTION_QUERY_KEY_TYPE_GENERATED_PRIMARY_KEY:'GeneratedPrimaryKey'; //Validat
RESOLUTION_QUERY_KEY_TYPE_SUPPLIED_PRIMARY_KEY: 'SuppliedPrimaryKey'; //Validated against equality key to ensure an actuial PK and create if don't find match
RESOLUTION_QUERY_KEY_TYPE_ALTERNATE_KEY: 'AlternateKey'; //AlternateKey (In an AlternateKey is specified then at least one required in the input record or fail resolution). AlternateKey && (CurationModel field == Create) then the input source is attempting to create a new record (e.g. from UI) block if existing record found
RESOLUTION_QUERY_KEY_TYPE_OPTIONAL: 'Optional';
RESOLUTION_QUERY_FILTER: 'filter';

// -------------------------------------- PRECEDENCE RULES --------------------------------------
PRECEDENCE_RULES: 'precedenceRules';
Expand Down
3 changes: 3 additions & 0 deletions .../antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryParserGrammar.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,7 @@ resolutionQuery: BRACE_OPEN
| resolutionQueryKeyType
| resolutionQueryOptional
| resolutionQueryPrecedence
| resolutionQueryFilter
)*
BRACE_CLOSE
;
Expand All @@ -300,6 +301,8 @@ resolutionQueryOptional: RESOLUTION_QUERY_OPTIONAL COLON boolean
;
resolutionQueryPrecedence: PRECEDENCE COLON INTEGER SEMI_COLON
;
resolutionQueryFilter: RESOLUTION_QUERY_FILTER COLON lambdaFunction SEMI_COLON
;
// -------------------------------------- PRECEDENCE RULES--------------------------------------

precedenceRules: PRECEDENCE_RULES COLON
Expand Down
43 changes: 35 additions & 8 deletions ...e/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,14 @@
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.RecordSource;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.RecordSourceVisitor;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.AcquisitionProtocol;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.KafkaAcquisitionProtocol;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.authorization.Authorization;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.identity.CollectionEquality;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.identity.IdentityResolution;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.identity.IdentityResolutionVisitor;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.identity.ResolutionQuery;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.precedence.*;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.trigger.Trigger;
import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.Variable;
import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda;
import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
import org.finos.legend.pure.generated.*;
Expand All @@ -49,7 +49,6 @@
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Class;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enumeration;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Type;
import org.finos.legend.pure.m4.coreinstance.CoreInstance;

import java.util.*;
import java.util.stream.Collectors;
Expand All @@ -58,15 +57,13 @@
import static com.google.common.collect.Sets.newHashSet;
import static java.lang.String.format;
import static java.util.Objects.nonNull;
import static org.apache.commons.lang3.BooleanUtils.isFalse;
import static org.apache.commons.lang3.BooleanUtils.isTrue;

public class HelperMasterRecordDefinitionBuilder
{
private static final String MASTERY_PACKAGE_PREFIX = "meta::pure::mastery::metamodel";
private static final String ROOT = "Root";
private static final Set<String> CONDITIONAL_BLOCK_RULE_PREDICATE_INPUT = newHashSet("incoming", "current");
private static final IdentityResolutionBuilder IDENTITY_RESOLUTION_BUILDER = new IdentityResolutionBuilder();

private HelperMasterRecordDefinitionBuilder()
{
Expand Down Expand Up @@ -104,15 +101,21 @@ private static String determineFullPath(Type type)
return Iterate.makeString(deque, "", "::", "::" + type._name());
}

public static Root_meta_pure_mastery_metamodel_identity_IdentityResolution buildIdentityResolution(IdentityResolution identityResolution, CompileContext context)
public static Root_meta_pure_mastery_metamodel_identity_IdentityResolution buildIdentityResolution(IdentityResolution identityResolution, String modelClass, CompileContext context)
{
IDENTITY_RESOLUTION_BUILDER.context = context;
return identityResolution.accept(IDENTITY_RESOLUTION_BUILDER);
return identityResolution.accept(new IdentityResolutionBuilder(context, modelClass));
}

private static class IdentityResolutionBuilder implements IdentityResolutionVisitor<Root_meta_pure_mastery_metamodel_identity_IdentityResolution>
{
private CompileContext context;
private final CompileContext context;
private final String modelClass;

public IdentityResolutionBuilder(CompileContext context, String modelClass)
{
this.context = context;
this.modelClass = modelClass;
}

@Override
public Root_meta_pure_mastery_metamodel_identity_IdentityResolution visit(IdentityResolution protocolVal)
Expand All @@ -131,12 +134,36 @@ private Iterable<Root_meta_pure_mastery_metamodel_identity_ResolutionQuery> visi
resQuery._keyType(protocolQuery.keyType == null ? null : context.resolveEnumValue(KEY_TYPE_FULL_PATH, protocolQuery.keyType.name()));
resQuery._optional(protocolQuery.optional);
resQuery._precedence(protocolQuery.precedence);
if (protocolQuery.filter != null)
{
validateFilterInput(protocolQuery.filter);
resQuery._filter(HelperValueSpecificationBuilder.buildLambda(protocolQuery.filter, context));
}

ListIterate.forEachWithIndex(protocolQuery.queries, (lambda, i) ->
resQuery._queriesAdd(HelperValueSpecificationBuilder.buildLambda(lambda, context)));
list.add(resQuery);
return list;
}

private void validateFilterInput(Lambda predicate)
{
List<Variable> parameters = predicate.parameters;
if (parameters.size() != 1)
{
throw new EngineException(format("The resolution query filter must have exactly one parameter specified in the lambda function - found %s", parameters.size()), EngineErrorType.COMPILATION);
}
Variable parameter = parameters.get(0);
String parameterClass = parameter._class;
if (!modelClass.equals(parameterClass))
{
throw new EngineException(format("Input Class for the resolution key filter should be %s, however found %s", modelClass, parameterClass), EngineErrorType.COMPILATION);
}
if (parameter.multiplicity.lowerBound != 1 || !parameter.multiplicity.isUpperBoundEqualTo(1))
{
throw new EngineException("Expected input for resolution key filter to have multiplicity 1", EngineErrorType.COMPILATION);
}
}
}

public static RichIterable<Root_meta_pure_mastery_metamodel_RecordSource> buildRecordSources(List<RecordSource> recordSources, CompileContext context)
Expand Down
2 changes: 1 addition & 1 deletion ...egend/engine/language/pure/dsl/mastery/compiler/toPureGraph/MasteryCompilerExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public Iterable<? extends Processor<?>> getExtraProcessors()
(masterRecordDefinition, context) ->
{
Root_meta_pure_mastery_metamodel_MasterRecordDefinition pureMasteryMetamodelMasterRecordDefinition = (Root_meta_pure_mastery_metamodel_MasterRecordDefinition) context.pureModel.getOrCreatePackage(masterRecordDefinition._package)._children().detect(c -> masterRecordDefinition.name.equals(c._name()));
pureMasteryMetamodelMasterRecordDefinition._identityResolution(HelperMasterRecordDefinitionBuilder.buildIdentityResolution(masterRecordDefinition.identityResolution, context));
pureMasteryMetamodelMasterRecordDefinition._identityResolution(HelperMasterRecordDefinitionBuilder.buildIdentityResolution(masterRecordDefinition.identityResolution, masterRecordDefinition.modelClass, context));
pureMasteryMetamodelMasterRecordDefinition._sources(HelperMasterRecordDefinitionBuilder.buildRecordSources(masterRecordDefinition.sources, context));
pureMasteryMetamodelMasterRecordDefinition._postCurationEnrichmentService(BuilderUtil.buildService(masterRecordDefinition.postCurationEnrichmentService, context, masterRecordDefinition.sourceInformation));
pureMasteryMetamodelMasterRecordDefinition._exceptionWorkflowTransformService(BuilderUtil.buildService(masterRecordDefinition.exceptionWorkflowTransformService, context, masterRecordDefinition.sourceInformation));
Expand Down
7 changes: 7 additions & 0 deletions ...rg/finos/legend/engine/language/pure/dsl/mastery/grammar/from/MasteryParseTreeWalker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -736,6 +736,13 @@ private ResolutionQuery visitResolutionQuery(MasteryParserGrammar.ResolutionQuer
MasteryParserGrammar.ResolutionQueryPrecedenceContext resolutionQueryPrecedenceContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.resolutionQueryPrecedence(), "precedence", sourceInformation);
resolutionQuery.precedence = Integer.parseInt(resolutionQueryPrecedenceContext.INTEGER().getText());

//filter
MasteryParserGrammar.ResolutionQueryFilterContext resolutionQueryFilterContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.resolutionQueryFilter(), "filter", sourceInformation);
if (resolutionQueryFilterContext != null)
{
resolutionQuery.filter = visitLambda(resolutionQueryFilterContext.lambdaFunction());
}

return resolutionQuery;
}

Expand Down
4 changes: 4 additions & 0 deletions ...inos/legend/engine/language/pure/dsl/mastery/grammar/to/HelperMasteryGrammarComposer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,10 @@ private static String renderResolutionQueries(IdentityResolution identityResolut
builder.append(getTabString(indentLevel + 4)).append("optional: ").append(resolutionQuery.optional).append(";\n");
}
builder.append(getTabString(indentLevel + 4)).append("precedence: ").append(resolutionQuery.precedence).append(";\n");
if (resolutionQuery.filter != null)
{
builder.append(getTabString(indentLevel + 4)).append("filter: ").append(lambdaToString(resolutionQuery.filter, context)).append(";\n");
}
builder.append(getTabString(indentLevel + 3)).append("}");
});
builder.append("\n").append(getTabString(indentLevel + 2)).append("]");
Expand Down
129 changes: 129 additions & 0 deletions ...end/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,15 @@ public class TestMasteryCompilationFromGrammar extends TestCompilationFromGramma
" keyType: AlternateKey;\n" +
" optional: true;\n" +
" precedence: 2;\n" +
" filter: {input: org::dataeng::Widget[1]|$input.trigger == 'A'};\n" +
" },\n" +
" {\n" +
" queries: [ {input: org::dataeng::Widget[1],EFFECTIVE_DATE: StrictDate[1]|org::dataeng::Widget.all()->filter(widget|((($widget.identifiers.identifierType == 'CUSIP') && ($input.identifiers->filter(idType|$idType.identifierType == 'CUSIP').identifier == $widget.identifiers->filter(idType|$idType.identifierType == 'CUSIP').identifier)) && ($widget.identifiers.FROM_Z->toOne() <= $EFFECTIVE_DATE)) && ($widget.identifiers.THRU_Z->toOne() > $EFFECTIVE_DATE))}\n" +
" ];\n" +
" keyType: AlternateKey;\n" +
" optional: true;\n" +
" precedence: 2;\n" +
" filter: {input: org::dataeng::Widget[1]|$input.trigger == 'B'};\n" +
" },\n" +
" {\n" +
" queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.trigger == $input.trigger)}\n" +
Expand Down Expand Up @@ -1398,6 +1407,126 @@ public void testCompilationErrorWhenConditionalRuleHasScopeDefined()
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [23:5-29:5]: ConditionalRule with ruleScope is currently unsupported");
}

@Test
public void testCompilationErrorWhenResolutionFilterHasMoreThanOneParameter()
{
String model = "###Pure\n" +
"Class org::dataeng::Widget\n" +
"{\n" +
" widgetId: String[0..1];\n" +
"}\n\n" +
"###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" +
"\n" +
"{\n" +
" modelClass: org::dataeng::Widget;\n" +
" identityResolution: \n" +
" {\n" +
" resolutionQueries:\n" +
" [\n" +
" {\n" +
" queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" +
" ];\n" +
" precedence: 1;\n" +
" filter: {input:org::dataeng::Widget[1],current:org::dataeng::Widget[1]| $input.widgetId == $current.widget};\n" +
" }\n" +
" ]\n" +
" }\n" +
" recordSources:\n" +
" [\n" +
" widget-producer: {\n" +
" description: 'REST Acquisition source.';\n" +
" status: Development;\n" +
" recordService: {\n" +
" acquisitionProtocol: REST;\n" +
" };\n" +
" trigger: Manual;\n" +
" }\n" +
" ]\n" +
"}\n\n";

TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [8:1-34:1]: Error in 'alloy::mastery::WidgetMasterRecord': The resolution query filter must have exactly one parameter specified in the lambda function - found 2");
}

@Test
public void testCompilationErrorWhenResolutionFilterHasWrongParameterType()
{
String model = "###Pure\n" +
"Class org::dataeng::Widget\n" +
"{\n" +
" widgetId: String[0..1];\n" +
"}\n\n" +
"###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" +
"\n" +
"{\n" +
" modelClass: org::dataeng::Widget;\n" +
" identityResolution: \n" +
" {\n" +
" resolutionQueries:\n" +
" [\n" +
" {\n" +
" queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" +
" ];\n" +
" precedence: 1;\n" +
" filter: {input:String[1]| $input == 'A'};\n" +
" }\n" +
" ]\n" +
" }\n" +
" recordSources:\n" +
" [\n" +
" widget-producer: {\n" +
" description: 'REST Acquisition source.';\n" +
" status: Development;\n" +
" recordService: {\n" +
" acquisitionProtocol: REST;\n" +
" };\n" +
" trigger: Manual;\n" +
" }\n" +
" ]\n" +
"}\n\n";

TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [8:1-34:1]: Error in 'alloy::mastery::WidgetMasterRecord': Input Class for the resolution key filter should be org::dataeng::Widget, however found String");
}

@Test
public void testCompilationErrorWhenResolutionFilterHasWrongParameterMultiplicity()
{
String model = "###Pure\n" +
"Class org::dataeng::Widget\n" +
"{\n" +
" widgetId: String[0..1];\n" +
"}\n\n" +
"###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" +
"\n" +
"{\n" +
" modelClass: org::dataeng::Widget;\n" +
" identityResolution: \n" +
" {\n" +
" resolutionQueries:\n" +
" [\n" +
" {\n" +
" queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" +
" ];\n" +
" precedence: 1;\n" +
" filter: {input:org::dataeng::Widget[0..1]| $input.widgetId == 'A'};\n" +
" }\n" +
" ]\n" +
" }\n" +
" recordSources:\n" +
" [\n" +
" widget-producer: {\n" +
" description: 'REST Acquisition source.';\n" +
" status: Development;\n" +
" recordService: {\n" +
" acquisitionProtocol: REST;\n" +
" };\n" +
" trigger: Manual;\n" +
" }\n" +
" ]\n" +
"}\n\n";

TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [8:1-34:1]: Error in 'alloy::mastery::WidgetMasterRecord': Expected input for resolution key filter to have multiplicity 1");
}

private void assertDataProviders(PureModel model)
{
PackageableElement lseDataProvider = model.getPackageableElement("alloy::mastery::dataprovider::LSE");
Expand Down
1 change: 1 addition & 0 deletions ...nd/engine/protocol/pure/v1/model/packageableElement/mastery/identity/ResolutionQuery.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,5 @@ public class ResolutionQuery
public ResolutionKeyType keyType;
public Boolean optional;
public Integer precedence;
public Lambda filter;
}
3 changes: 3 additions & 0 deletions ...tery/legend-engine-xt-mastery-pure/src/main/resources/core_mastery/mastery/metamodel.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,9 @@ meta::pure::mastery::metamodel::identity::ResolutionQuery

{doc.doc='The query precedence applied when there are more than one ResoultionQuery definitions on an IdentityResolution, 1 is hiughest.'}
precedence : Integer[1];

{doc.doc='An optional filter that when specified will determine whether the input should execute this resolution query or not.'}
filter: meta::pure::metamodel::function::LambdaFunction<{Any[*]->Boolean[1]}>[0..1];
}

Enum {doc.doc = 'Types of resolution keys.'}
Expand Down

0 comments on commit 3dd8572

Please sign in to comment.