From f864e1549f1d72df275efd9c72b74b0dfb419fba Mon Sep 17 00:00:00 2001 From: Arik Pamnani Date: Fri, 6 Oct 2023 00:39:53 +0530 Subject: [PATCH] Add few optional fields in mastery spec (#2347) --- .../pure/grammar/from/antlr4/MasteryLexerGrammar.g4 | 1 + .../pure/grammar/from/antlr4/MasteryParserGrammar.g4 | 3 +++ .../HelperMasterRecordDefinitionBuilder.java | 3 ++- .../mastery/grammar/from/MasteryParseTreeWalker.java | 11 +++++++++-- .../grammar/to/HelperMasteryGrammarComposer.java | 9 ++++++++- .../test/TestMasteryCompilationFromGrammar.java | 9 +++++++-- .../packageableElement/mastery/RecordSource.java | 4 ++-- .../mastery/identity/ResolutionQuery.java | 2 +- .../resources/core_mastery/mastery/metamodel.pure | 5 ++++- 9 files changed, 37 insertions(+), 10 deletions(-) diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryLexerGrammar.g4 b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryLexerGrammar.g4 index 306e56dae49..7a15d60935f 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryLexerGrammar.g4 +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryLexerGrammar.g4 @@ -52,6 +52,7 @@ IDENTITY_RESOLUTION: 'identityResolution'; RESOLUTION_QUERIES: 'resolutionQueries'; RESOLUTION_QUERY_EXPRESSIONS: 'queries'; RESOLUTION_QUERY_KEY_TYPE: 'keyType'; +RESOLUTION_QUERY_OPTIONAL: 'optional'; RESOLUTION_QUERY_KEY_TYPE_GENERATED_PRIMARY_KEY:'GeneratedPrimaryKey'; //Validated against equality key to ensure an acrual PK and fail if don't find match RESOLUTION_QUERY_KEY_TYPE_SUPPLIED_PRIMARY_KEY: 'SuppliedPrimaryKey'; //Validated against equality key to ensure an actuial PK and create if don't find match RESOLUTION_QUERY_KEY_TYPE_ALTERNATE_KEY: 'AlternateKey'; //AlternateKey (In an AlternateKey is specified then at least one required in the input record or fail resolution). AlternateKey && (CurationModel field == Create) then the input source is attempting to create a new record (e.g. from UI) block if existing record found diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryParserGrammar.g4 b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryParserGrammar.g4 index 98379331ac0..b039d6becae 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryParserGrammar.g4 +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryParserGrammar.g4 @@ -275,6 +275,7 @@ resolutionQueries: RESOLUTION_QUERIES COLON resolutionQuery: BRACE_OPEN (queryExpressions | resolutionQueryKeyType + | resolutionQueryOptional | resolutionQueryPrecedence )* BRACE_CLOSE @@ -295,6 +296,8 @@ resolutionQueryKeyType: RESOLUTION_QUERY_KEY_TYPE COLON ( ) SEMI_COLON ; +resolutionQueryOptional: RESOLUTION_QUERY_OPTIONAL COLON boolean_value SEMI_COLON +; resolutionQueryPrecedence: PRECEDENCE COLON INTEGER SEMI_COLON ; // -------------------------------------- PRECEDENCE RULES-------------------------------------- diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java index bfc9ba0a848..db5a9ebe315 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java @@ -120,7 +120,8 @@ private Iterable visi Root_meta_pure_mastery_metamodel_identity_ResolutionQuery resQuery = new Root_meta_pure_mastery_metamodel_identity_ResolutionQuery_Impl(""); String KEY_TYPE_FULL_PATH = MASTERY_PACKAGE_PREFIX + "::identity::ResolutionKeyType"; - resQuery._keyType(context.resolveEnumValue(KEY_TYPE_FULL_PATH, protocolQuery.keyType.name())); + resQuery._keyType(protocolQuery.keyType == null ? null : context.resolveEnumValue(KEY_TYPE_FULL_PATH, protocolQuery.keyType.name())); + resQuery._optional(protocolQuery.optional); resQuery._precedence(protocolQuery.precedence); ListIterate.forEachWithIndex(protocolQuery.queries, (lambda, i) -> diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/MasteryParseTreeWalker.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/MasteryParseTreeWalker.java index 3ab85c09c89..8a252c6b062 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/MasteryParseTreeWalker.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/MasteryParseTreeWalker.java @@ -719,8 +719,15 @@ private ResolutionQuery visitResolutionQuery(MasteryParserGrammar.ResolutionQuer resolutionQuery.queries = ListIterate.flatCollect(ctx.queryExpressions(), this::visitQueryExpressions); //keyType - MasteryParserGrammar.ResolutionQueryKeyTypeContext resolutionQueryKeyTypeContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.resolutionQueryKeyType(), "keyType", sourceInformation); - resolutionQuery.keyType = visitResolutionKeyType(resolutionQueryKeyTypeContext); + MasteryParserGrammar.ResolutionQueryKeyTypeContext resolutionQueryKeyTypeContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.resolutionQueryKeyType(), "keyType", sourceInformation); + if (resolutionQueryKeyTypeContext != null) + { + resolutionQuery.keyType = visitResolutionKeyType(resolutionQueryKeyTypeContext); + } + + //optional + MasteryParserGrammar.ResolutionQueryOptionalContext resolutionQueryOptionalContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.resolutionQueryOptional(), "optional", sourceInformation); + resolutionQuery.optional = evaluateBoolean(resolutionQueryOptionalContext, (resolutionQueryOptionalContext != null ? resolutionQueryOptionalContext.boolean_value() : null), null); //precedence - Field 'precedence' should be specified only once MasteryParserGrammar.ResolutionQueryPrecedenceContext resolutionQueryPrecedenceContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.resolutionQueryPrecedence(), "precedence", sourceInformation); diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/to/HelperMasteryGrammarComposer.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/to/HelperMasteryGrammarComposer.java index f7a4726e3e0..38aace1908c 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/to/HelperMasteryGrammarComposer.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/to/HelperMasteryGrammarComposer.java @@ -445,7 +445,14 @@ private static String renderResolutionQueries(IdentityResolution identityResolut builder.append("\n").append(getTabString(indentLevel + 4)).append("queries: [ "); builder.append(renderQueries(resolutionQuery, indentLevel + 5, context)); builder.append(getTabString(indentLevel + 4)).append(" ];\n"); - builder.append(getTabString(indentLevel + 4)).append("keyType: ").append(resolutionQuery.keyType).append(";\n"); + if (resolutionQuery.keyType != null) + { + builder.append(getTabString(indentLevel + 4)).append("keyType: ").append(resolutionQuery.keyType).append(";\n"); + } + if (resolutionQuery.optional != null) + { + builder.append(getTabString(indentLevel + 4)).append("optional: ").append(resolutionQuery.optional).append(";\n"); + } builder.append(getTabString(indentLevel + 4)).append("precedence: ").append(resolutionQuery.precedence).append(";\n"); builder.append(getTabString(indentLevel + 3)).append("}"); }); diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java index 0850ab66f96..b9953070433 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java @@ -113,18 +113,21 @@ public class TestMasteryCompilationFromGrammar extends TestCompilationFromGramma " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + " ];\n" + " keyType: GeneratedPrimaryKey;\n" + + " optional: true;\n" + " precedence: 1;\n" + " },\n" + " {\n" + " queries: [ {input: org::dataeng::Widget[1],EFFECTIVE_DATE: StrictDate[1]|org::dataeng::Widget.all()->filter(widget|((($widget.identifiers.identifierType == 'ISIN') && ($input.identifiers->filter(idType|$idType.identifierType == 'ISIN').identifier == $widget.identifiers->filter(idType|$idType.identifierType == 'ISIN').identifier)) && ($widget.identifiers.FROM_Z->toOne() <= $EFFECTIVE_DATE)) && ($widget.identifiers.THRU_Z->toOne() > $EFFECTIVE_DATE))}\n" + " ];\n" + " keyType: AlternateKey;\n" + + " optional: true;\n" + " precedence: 2;\n" + " },\n" + " {\n" + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.trigger == $input.trigger)}\n" + " ];\n" + " keyType: Optional;\n" + + " optional: true;\n" + " precedence: 3;\n" + " }\n" + " ]\n" + @@ -393,7 +396,6 @@ public class TestMasteryCompilationFromGrammar extends TestCompilationFromGramma " {\n" + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + " ];\n" + - " keyType: GeneratedPrimaryKey;\n" + " precedence: 1;\n" + " }\n" + " ]\n" + @@ -477,6 +479,7 @@ public class TestMasteryCompilationFromGrammar extends TestCompilationFromGramma " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + " ];\n" + " keyType: GeneratedPrimaryKey;\n" + + " optional: true;\n" + " precedence: 1;\n" + " }" + " ]\n" + @@ -550,10 +553,12 @@ public void testMasteryFullModel() Object[] queriesArray = idRes._resolutionQueries().toArray(); assertEquals(1, ((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._precedence()); assertEquals("GeneratedPrimaryKey", ((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._keyType()._name()); + assertTrue(((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._optional()); assertResolutionQueryLambdas(((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._queries().toList()); assertEquals(2, ((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[1])._precedence()); assertEquals("AlternateKey", ((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[1])._keyType()._name()); + assertTrue(((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._optional()); assertResolutionQueryLambdas(((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._queries().toList()); //PrecedenceRule @@ -1000,6 +1005,6 @@ protected String getDuplicatedElementTestCode() @Override public String getDuplicatedElementTestExpectedErrorMessage() { - return "COMPILATION error at [8:1-35:1]: Duplicated element 'org::dataeng::Widget'"; + return "COMPILATION error at [8:1-36:1]: Duplicated element 'org::dataeng::Widget'"; } } \ No newline at end of file diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/RecordSource.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/RecordSource.java index 19c950d2ef6..3c9500f0522 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/RecordSource.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/RecordSource.java @@ -26,8 +26,8 @@ public class RecordSource { public String parseService; public String transformService; - public List tags = new ArrayList(); - public List partitions = Collections.emptyList(); + public List tags; + public List partitions; public String id; public String description; public RecordSourceStatus status; diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/identity/ResolutionQuery.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/identity/ResolutionQuery.java index abfc2f1b099..f2e0b634dfc 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/identity/ResolutionQuery.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/identity/ResolutionQuery.java @@ -20,8 +20,8 @@ public class ResolutionQuery { - //public List queries; public List queries; public ResolutionKeyType keyType; + public Boolean optional; public Integer precedence; } diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/src/main/resources/core_mastery/mastery/metamodel.pure b/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/src/main/resources/core_mastery/mastery/metamodel.pure index 318972add4d..f989941b7b4 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/src/main/resources/core_mastery/mastery/metamodel.pure +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/src/main/resources/core_mastery/mastery/metamodel.pure @@ -171,7 +171,10 @@ meta::pure::mastery::metamodel::identity::ResolutionQuery queries : meta::pure::metamodel::function::LambdaFunction<{Any[1],StrictDate[0..1]->Any[*]}>[1..*]; //TODO in compiler check that parameter and return type are of same type, via subclass of LambdaFunction? {doc.doc='The key type used to define influences how to generate queries issued against the store and validate the results of query execution.'} - keyType : meta::pure::mastery::metamodel::identity::ResolutionKeyType[1]; + keyType : meta::pure::mastery::metamodel::identity::ResolutionKeyType[0..1]; + + {doc.doc='Field to identify if key is optional.'} + optional : Boolean[0..1]; {doc.doc='The query precedence applied when there are more than one ResoultionQuery definitions on an IdentityResolution, 1 is hiughest.'} precedence : Integer[1];