diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperAcquisitionBuilder.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperAcquisitionBuilder.java index 885389da4a0..d341a969673 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperAcquisitionBuilder.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperAcquisitionBuilder.java @@ -21,6 +21,7 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.DESDecryption; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.Decryption; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.FileAcquisitionProtocol; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.FileType; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.KafkaAcquisitionProtocol; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.LegendServiceAcquisitionProtocol; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.PGPDecryption; @@ -42,6 +43,7 @@ import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; import static java.lang.String.format; +import static org.apache.commons.lang3.StringUtils.isEmpty; public class HelperAcquisitionBuilder { @@ -75,6 +77,7 @@ public static Root_meta_pure_mastery_metamodel_acquisition_AcquisitionProtocol b public static Root_meta_pure_mastery_metamodel_acquisition_FileAcquisitionProtocol buildFileAcquisitionProtocol(FileAcquisitionProtocol acquisitionProtocol, CompileContext context) { + validateFileAcquisitionProtocol(acquisitionProtocol); Root_meta_pure_mastery_metamodel_connection_FileConnection fileConnection; PackageableElement packageableElement = context.resolvePackageableElement(acquisitionProtocol.connection, acquisitionProtocol.sourceInformation); if (packageableElement instanceof Root_meta_pure_mastery_metamodel_connection_FileConnection) @@ -98,6 +101,15 @@ public static Root_meta_pure_mastery_metamodel_acquisition_FileAcquisitionProtoc ._decryption(acquisitionProtocol.decryption == null ? null : buildDecryption(acquisitionProtocol.decryption, context)); } + private static void validateFileAcquisitionProtocol(FileAcquisitionProtocol fileAcquisitionProtocol) + { + if (fileAcquisitionProtocol.fileType == FileType.JSON && isEmpty(fileAcquisitionProtocol.recordsKey)) + { + throw new EngineException("'recordsKey' must be specified when file type is JSON", fileAcquisitionProtocol.sourceInformation, EngineErrorType.COMPILATION); + } + } + + public static Root_meta_pure_mastery_metamodel_acquisition_file_Decryption buildDecryption(Decryption decryption, CompileContext context) { if (decryption instanceof PGPDecryption) diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java index db5a9ebe315..670bed7b19c 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java @@ -26,10 +26,12 @@ import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.MasterRecordDefinition; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.Profile; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.RecordService; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.RecordSource; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.RecordSourceVisitor; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.AcquisitionProtocol; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.KafkaAcquisitionProtocol; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.authorization.Authorization; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.identity.CollectionEquality; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.identity.IdentityResolution; @@ -50,8 +52,12 @@ import java.util.*; import java.util.stream.Collectors; +import static com.google.common.collect.Iterables.isEmpty; import static com.google.common.collect.Sets.newHashSet; import static java.lang.String.format; +import static java.util.Objects.nonNull; +import static org.apache.commons.lang3.BooleanUtils.isFalse; +import static org.apache.commons.lang3.BooleanUtils.isTrue; public class HelperMasterRecordDefinitionBuilder { @@ -167,6 +173,7 @@ public Root_meta_pure_mastery_metamodel_precedence_PrecedenceRule visit(Preceden } else if (precedenceRule instanceof DeleteRule) { + validateNoDataProviderScope((DeleteRule) precedenceRule); purePrecedenceRule = new Root_meta_pure_mastery_metamodel_precedence_DeleteRule_Impl(""); } else if (precedenceRule instanceof CreateRule) @@ -175,7 +182,7 @@ else if (precedenceRule instanceof CreateRule) } else if (precedenceRule instanceof ConditionalRule) { - purePrecedenceRule = visitConditionalRule(precedenceRule); + purePrecedenceRule = visitConditionalRule((ConditionalRule) precedenceRule); } else { @@ -218,10 +225,10 @@ private Root_meta_pure_mastery_metamodel_precedence_SourcePrecedenceRule visitSo return pureSourcePrecedenceRule; } - private Root_meta_pure_mastery_metamodel_precedence_ConditionalRule visitConditionalRule(PrecedenceRule precedenceRule) + private Root_meta_pure_mastery_metamodel_precedence_ConditionalRule visitConditionalRule(ConditionalRule conditionalRule) { + validateNoScopeSet(conditionalRule); Root_meta_pure_mastery_metamodel_precedence_ConditionalRule pureConditionalRule = new Root_meta_pure_mastery_metamodel_precedence_ConditionalRule_Impl(""); - ConditionalRule conditionalRule = (ConditionalRule) precedenceRule; validatePredicateInput(conditionalRule.predicate); pureConditionalRule._predicate(HelperValueSpecificationBuilder.buildLambda(conditionalRule.predicate, context)); return pureConditionalRule; @@ -234,6 +241,26 @@ private void validatePredicateInput(Lambda predicate) validateInputMultiplicity(predicate); } + private void validateNoScopeSet(ConditionalRule conditionalRule) + { + if (!isEmpty(conditionalRule.scopes)) + { + throw new EngineException( + "ConditionalRule with ruleScope is currently unsupported", conditionalRule.sourceInformation, + EngineErrorType.COMPILATION); + } + } + + private void validateNoDataProviderScope(DeleteRule deleteRule) + { + if (!isEmpty(deleteRule.scopes) && deleteRule.scopes.stream().anyMatch(scope -> scope instanceof DataProviderTypeScope)) + { + throw new EngineException( + "DataProviderTypeScope is not allowed on DeleteRule", deleteRule.sourceInformation, + EngineErrorType.COMPILATION); + } + } + private void validateInputVariableNames(Lambda predicate) { Set actualNames = predicate.parameters.stream().map(variable -> variable.name).collect(Collectors.toSet()); @@ -359,6 +386,7 @@ public RecordSourceBuilder(CompileContext context) @Override public Root_meta_pure_mastery_metamodel_RecordSource visit(RecordSource protocolSource) { + validateRecordSource(protocolSource); List extensions = IMasteryCompilerExtension.getExtensions(); List> processors = ListIterate.flatCollect(extensions, IMasteryCompilerExtension::getExtraAuthorizationProcessors); List> triggerProcessors = ListIterate.flatCollect(extensions, IMasteryCompilerExtension::getExtraTriggerProcessors); @@ -415,6 +443,25 @@ private static RichIterable 31) + { + throw new EngineException(format("Invalid record source id '%s'; id must not be longer than 31 characters.", recordSource.id), recordSource.sourceInformation, EngineErrorType.COMPILATION); + } + + boolean kafkaSource = nonNull(recordSource.recordService) && nonNull(recordSource.recordService.acquisitionProtocol) && recordSource.recordService.acquisitionProtocol.isKafkaAcquisitionProtocol(); + + if (isTrue(recordSource.sequentialData) && kafkaSource && nonNull(recordSource.runProfile) && recordSource.runProfile != Profile.ExtraSmall) + { + throw new EngineException("'runProfile' can only be set to ExtraSmall for Delta kafka sources", recordSource.sourceInformation, EngineErrorType.COMPILATION); + } + if (kafkaSource && nonNull(recordSource.runProfile) && recordSource.runProfile != Profile.Small) + { + throw new EngineException("'runProfile' can only be set to Small for Full Universe kafka sources", recordSource.sourceInformation, EngineErrorType.COMPILATION); + } + } } private static Root_meta_pure_mastery_metamodel_DataProvider getAndValidateDataProvider(String path, SourceInformation sourceInformation, CompileContext context) diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperTriggerBuilder.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperTriggerBuilder.java index fc3172ab889..9f68f4c0b67 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperTriggerBuilder.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperTriggerBuilder.java @@ -16,14 +16,19 @@ import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.engine.language.pure.compiler.toPureGraph.CompileContext; +import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.trigger.CronTrigger; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.trigger.Frequency; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.trigger.ManualTrigger; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.trigger.Trigger; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; import org.finos.legend.pure.generated.Root_meta_pure_mastery_metamodel_trigger_CronTrigger; import org.finos.legend.pure.generated.Root_meta_pure_mastery_metamodel_trigger_CronTrigger_Impl; import org.finos.legend.pure.generated.Root_meta_pure_mastery_metamodel_trigger_ManualTrigger_Impl; import org.finos.legend.pure.generated.Root_meta_pure_mastery_metamodel_trigger_Trigger; +import static org.eclipse.collections.impl.utility.Iterate.isEmpty; + public class HelperTriggerBuilder { @@ -45,6 +50,7 @@ public static Root_meta_pure_mastery_metamodel_trigger_Trigger buildTrigger(Trig private static Root_meta_pure_mastery_metamodel_trigger_CronTrigger buildCronTrigger(CronTrigger cronTrigger, CompileContext context) { + validateCronTrigger(cronTrigger); return new Root_meta_pure_mastery_metamodel_trigger_CronTrigger_Impl("") ._minute(cronTrigger.minute) ._hour(cronTrigger.hour) @@ -55,4 +61,32 @@ private static Root_meta_pure_mastery_metamodel_trigger_CronTrigger buildCronTri ._month(cronTrigger.year == null ? null : context.resolveEnumValue("meta::pure::mastery::metamodel::trigger::Month", cronTrigger.month.name())) ._days(ListIterate.collect(cronTrigger.days, day -> context.resolveEnumValue("meta::pure::mastery::metamodel::trigger::Day", day.name()))); } + + private static void validateCronTrigger(CronTrigger cronTrigger) + { + if ((cronTrigger.frequency == Frequency.Intraday || cronTrigger.frequency == Frequency.Daily) && isEmpty(cronTrigger.days)) + { + throw new EngineException("'days' must not be empty when trigger frequency is Daily or Intraday", cronTrigger.sourceInformation, EngineErrorType.COMPILATION); + } + + if (cronTrigger.frequency == Frequency.Weekly && cronTrigger.days.size() != 1) + { + throw new EngineException("'days' specified must be exactly one when trigger frequency is Weekly", cronTrigger.sourceInformation, EngineErrorType.COMPILATION); + } + + if (!isInHourRange(cronTrigger.hour) || !isInMinuteRange(cronTrigger.minute)) + { + throw new EngineException("'hour' must be a number between 0 and 23 (both inclusive), and 'minute' must be a number between 0 and 59 (both inclusive)", cronTrigger.sourceInformation, EngineErrorType.COMPILATION); + } + } + + private static boolean isInHourRange(Integer hour) + { + return 0 <= hour && hour < 24; + } + + private static boolean isInMinuteRange(Integer minute) + { + return 0 <= minute && minute < 60; + } } diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/trigger/TriggerParseTreeWalker.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/trigger/TriggerParseTreeWalker.java index 3d39052465e..506b8efca39 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/trigger/TriggerParseTreeWalker.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/trigger/TriggerParseTreeWalker.java @@ -93,7 +93,7 @@ private Trigger visitCronTrigger(TriggerParserGrammar.CronTriggerContext ctx) // frequency - TriggerParserGrammar.FrequencyContext frequencyContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.frequency(), "frequency", sourceInformation); + TriggerParserGrammar.FrequencyContext frequencyContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.frequency(), "frequency", sourceInformation); if (frequencyContext != null) { String frequencyString = frequencyContext.frequencyValue().getText(); @@ -101,7 +101,7 @@ private Trigger visitCronTrigger(TriggerParserGrammar.CronTriggerContext ctx) } // days - TriggerParserGrammar.DaysContext daysContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.days(), "days", sourceInformation); + TriggerParserGrammar.DaysContext daysContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.days(), "days", sourceInformation); if (daysContext != null) { cronTrigger.days = ListIterate.collect(daysContext.dayValue(), this::visitRunDay); diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java index b9953070433..0b6c2a75497 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java @@ -903,6 +903,398 @@ public void testMasteryDeprecatedModelCanStillCompile() assertEquals("Widget", masterRecordDefinition._modelClass()._name()); } + @Test + public void testCompilationErrorWhenInvalidTriggerDefinition() + { + String model = "###Pure\n" + + "Class org::dataeng::Widget\n" + + "{\n" + + " widgetId: String[0..1];\n" + + "}\n\n" + + "###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" + + "\n" + + "{\n" + + " modelClass: org::dataeng::Widget;\n" + + " identityResolution: \n" + + " {\n" + + " resolutionQueries:\n" + + " [\n" + + " {\n" + + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + + " ];\n" + + " precedence: 1;\n" + + " }\n" + + " ]\n" + + " }\n" + + " recordSources:\n" + + " [\n" + + " widget-producer: {\n" + + " description: 'REST Acquisition source.';\n" + + " status: Development;\n" + + " recordService: {\n" + + " acquisitionProtocol: REST;\n" + + " };\n" + + " trigger: Cron #{\n" + + " minute: 70;\n" + + " hour: 25;\n" + + " timezone: 'UTC';\n" + + " frequency: Daily;\n" + + " days: [ Monday, Tuesday, Wednesday, Thursday, Friday ];\n" + + " }#;\n" + + " }\n" + + " ]\n" + + "}\n"; + + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [8:1-39:1]: Error in 'alloy::mastery::WidgetMasterRecord': 'hour' must be a number between 0 and 23 (both inclusive), and 'minute' must be a number between 0 and 59 (both inclusive)"); + } + + @Test + public void testCompilationErrorWhenWeeklyFrequencyButMoreThanOneRunDaySpecified() + { + String model = "###Pure\n" + + "Class org::dataeng::Widget\n" + + "{\n" + + " widgetId: String[0..1];\n" + + "}\n\n" + + "###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" + + "\n" + + "{\n" + + " modelClass: org::dataeng::Widget;\n" + + " identityResolution: \n" + + " {\n" + + " resolutionQueries:\n" + + " [\n" + + " {\n" + + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + + " ];\n" + + " precedence: 1;\n" + + " }\n" + + " ]\n" + + " }\n" + + " recordSources:\n" + + " [\n" + + " widget-producer: {\n" + + " description: 'REST Acquisition source.';\n" + + " status: Development;\n" + + " recordService: {\n" + + " acquisitionProtocol: REST;\n" + + " };\n" + + " trigger: Cron #{\n" + + " minute: 45;\n" + + " hour: 2;\n" + + " timezone: 'UTC';\n" + + " frequency: Weekly;\n" + + " days: [ Monday, Tuesday ];\n" + + " }#;\n" + + " }\n" + + " ]\n" + + "}\n"; + + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [8:1-39:1]: Error in 'alloy::mastery::WidgetMasterRecord': 'days' specified must be exactly one when trigger frequency is Weekly"); + } + + @Test + public void testCompilationErrorWhenRecordSourceIdExceedThirtyOne() + { + String model = "###Pure\n" + + "Class org::dataeng::Widget\n" + + "{\n" + + " widgetId: String[0..1];\n" + + "}\n\n" + + "###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" + + "\n" + + "{\n" + + " modelClass: org::dataeng::Widget;\n" + + " identityResolution: \n" + + " {\n" + + " resolutionQueries:\n" + + " [\n" + + " {\n" + + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + + " ];\n" + + " precedence: 1;\n" + + " }\n" + + " ]\n" + + " }\n" + + " recordSources:\n" + + " [\n" + + " widget-producer-alloy-mastery-exceed-allowed-length: {\n" + + " description: 'REST Acquisition source.';\n" + + " status: Development;\n" + + " recordService: {\n" + + " acquisitionProtocol: REST;\n" + + " };\n" + + " trigger: Manual;\n" + + " }\n" + + " ]\n" + + "}\n"; + + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [24:5-31:5]: Invalid record source id 'widget-producer-alloy-mastery-exceed-allowed-length'; id must not be longer than 31 characters."); + } + + @Test + public void testCompilationErrorWhenDeltaKafkaSourceHasRunProfileOtherThanExtraSmall() + { + String model = "###Pure\n" + + "Class org::dataeng::Widget\n" + + "{\n" + + " widgetId: String[0..1];\n" + + "}\n\n" + + "###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" + + "\n" + + "{\n" + + " modelClass: org::dataeng::Widget;\n" + + " identityResolution: \n" + + " {\n" + + " resolutionQueries:\n" + + " [\n" + + " {\n" + + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + + " ];\n" + + " precedence: 1;\n" + + " }\n" + + " ]\n" + + " }\n" + + " recordSources:\n" + + " [\n" + + " widget-kafka: {\n" + + " description: 'Kafka Acquisition source.';\n" + + " status: Development;\n" + + " recordService: {\n" + + " acquisitionProtocol: Kafka #{\n" + + " dataType: JSON;\n" + + " connection: alloy::mastery::connection::KafkaConnection;\n" + + " }#;\n" + + " };\n" + + " sequentialData: true;\n" + + " runProfile: Medium;\n" + + " trigger: Manual;\n" + + " }\n" + + " ]\n" + + "}\n\n" + + + "MasteryConnection alloy::mastery::connection::KafkaConnection\n" + + "{\n" + + " specification: Kafka #{\n" + + " topicName: 'my-topic-name';\n" + + " topicUrls: [\n" + + " 'some.url.com:2100',\n" + + " 'another.url.com:2100'\n" + + " ];\n" + + " }#;\n" + + "}"; + + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [24:5-36:5]: 'runProfile' can only be set to ExtraSmall for Delta kafka sources"); + } + + @Test + public void testCompilationErrorWhenFullUniverseKafkaSourceHasRunProfileOtherThanSmall() + { + String model = "###Pure\n" + + "Class org::dataeng::Widget\n" + + "{\n" + + " widgetId: String[0..1];\n" + + "}\n\n" + + "###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" + + "\n" + + "{\n" + + " modelClass: org::dataeng::Widget;\n" + + " identityResolution: \n" + + " {\n" + + " resolutionQueries:\n" + + " [\n" + + " {\n" + + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + + " ];\n" + + " precedence: 1;\n" + + " }\n" + + " ]\n" + + " }\n" + + " recordSources:\n" + + " [\n" + + " widget-kafka: {\n" + + " description: 'Kafka Acquisition source.';\n" + + " status: Development;\n" + + " recordService: {\n" + + " acquisitionProtocol: Kafka #{\n" + + " dataType: JSON;\n" + + " connection: alloy::mastery::connection::KafkaConnection;\n" + + " }#;\n" + + " };\n" + + " runProfile: Medium;\n" + + " trigger: Manual;\n" + + " }\n" + + " ]\n" + + "}\n\n" + + + "MasteryConnection alloy::mastery::connection::KafkaConnection\n" + + "{\n" + + " specification: Kafka #{\n" + + " topicName: 'my-topic-name';\n" + + " topicUrls: [\n" + + " 'some.url.com:2100',\n" + + " 'another.url.com:2100'\n" + + " ];\n" + + " }#;\n" + + "}"; + + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [24:5-35:5]: 'runProfile' can only be set to Small for Full Universe kafka sources"); + } + + @Test + public void testCompilationErrorWhenJsonFileAcquisitionHasRecordsKey() + { + String model = "###Pure\n" + + "Class org::dataeng::Widget\n" + + "{\n" + + " widgetId: String[0..1];\n" + + "}\n\n" + + "###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" + + "\n" + + "{\n" + + " modelClass: org::dataeng::Widget;\n" + + " identityResolution: \n" + + " {\n" + + " resolutionQueries:\n" + + " [\n" + + " {\n" + + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + + " ];\n" + + " precedence: 1;\n" + + " }\n" + + " ]\n" + + " }\n" + + " recordSources:\n" + + " [\n" + + " widget-kafka: {\n" + + " description: 'Kafka Acquisition source.';\n" + + " status: Development;\n" + + " recordService: {\n" + + " acquisitionProtocol: File #{\n" + + " fileType: JSON;\n" + + " filePath: '/download/day-file.json';\n" + + " headerLines: 0;\n" + + " connection: alloy::mastery::connection::HTTPConnection;\n" + + " }#;\n" + + " };\n" + + " trigger: Manual;\n" + + " }\n" + + " ]\n" + + "}\n\n" + + + "MasteryConnection alloy::mastery::connection::HTTPConnection\n" + + "{\n" + + " specification: HTTP #{\n" + + " url: 'https://some.url.com';\n" + + " proxy: {\n" + + " host: 'proxy.url.com';\n" + + " port: 85;\n" + + " };\n" + + " }#;\n" + + "}\n\n"; + + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [8:1-38:1]: Error in 'alloy::mastery::WidgetMasterRecord': 'recordsKey' must be specified when file type is JSON"); + } + + @Test + public void testCompilationErrorWhenDeleteRuleHasDataProviderScope() + { + String model = "###Pure\n" + + "Class org::dataeng::Widget\n" + + "{\n" + + " widgetId: String[0..1];\n" + + "}\n\n" + + "###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" + + "\n" + + "{\n" + + " modelClass: org::dataeng::Widget;\n" + + " identityResolution: \n" + + " {\n" + + " resolutionQueries:\n" + + " [\n" + + " {\n" + + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + + " ];\n" + + " precedence: 1;\n" + + " }\n" + + " ]\n" + + " }\n" + + " precedenceRules: [\n" + + " DeleteRule: {\n" + + " path: org::dataeng::Widget.widgetId;\n" + + " ruleScope: [\n" + + " DataProviderTypeScope {Exchange}\n" + + " ];\n" + + " }\n" + + "]\n" + + " recordSources:\n" + + " [\n" + + " widget-producer: {\n" + + " description: 'REST Acquisition source.';\n" + + " status: Development;\n" + + " recordService: {\n" + + " acquisitionProtocol: REST;\n" + + " };\n" + + " trigger: Manual;\n" + + " }\n" + + " ]\n" + + "}\n\n" + + + "ExchangeDataProvider alloy::mastery::dataprovider::LSE;\n\n\n"; + + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [23:5-28:5]: DataProviderTypeScope is not allowed on DeleteRule"); + } + + @Test + public void testCompilationErrorWhenConditionalRuleHasScopeDefined() + { + String model = "###Pure\n" + + "Class org::dataeng::Widget\n" + + "{\n" + + " widgetId: String[0..1];\n" + + "}\n\n" + + "###Mastery\n" + "MasterRecordDefinition alloy::mastery::WidgetMasterRecord" + + "\n" + + "{\n" + + " modelClass: org::dataeng::Widget;\n" + + " identityResolution: \n" + + " {\n" + + " resolutionQueries:\n" + + " [\n" + + " {\n" + + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + + " ];\n" + + " precedence: 1;\n" + + " }\n" + + " ]\n" + + " }\n" + + " precedenceRules: [\n" + + " ConditionalRule: {\n" + + " predicate: {incoming: org::dataeng::Widget[1],current: org::dataeng::Widget[1]|$incoming.widgetId == $current.widgetId};\n" + + " path: org::dataeng::Widget.widgetId;\n" + + " ruleScope: [\n" + + " DataProviderTypeScope {Exchange}\n" + + " ];\n" + + " }\n" + + "]\n" + + " recordSources:\n" + + " [\n" + + " widget-producer: {\n" + + " description: 'REST Acquisition source.';\n" + + " status: Development;\n" + + " recordService: {\n" + + " acquisitionProtocol: REST;\n" + + " };\n" + + " trigger: Manual;\n" + + " }\n" + + " ]\n" + + "}\n\n" + + + "ExchangeDataProvider alloy::mastery::dataprovider::LSE;\n\n\n"; + + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(model, "COMPILATION error at [23:5-29:5]: ConditionalRule with ruleScope is currently unsupported"); + } + private void assertDataProviders(PureModel model) { PackageableElement lseDataProvider = model.getPackageableElement("alloy::mastery::dataprovider::LSE"); diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/acquisition/AcquisitionProtocol.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/acquisition/AcquisitionProtocol.java index 9648ef04e15..8dc98ee33d4 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/acquisition/AcquisitionProtocol.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/acquisition/AcquisitionProtocol.java @@ -14,6 +14,7 @@ package org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.finos.legend.engine.protocol.pure.v1.model.SourceInformation; @@ -26,4 +27,10 @@ public T accept(AcquisitionProtocolVisitor visitor) { return visitor.visit(this); } + + @JsonIgnore + public boolean isKafkaAcquisitionProtocol() + { + return false; + } } diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/acquisition/KafkaAcquisitionProtocol.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/acquisition/KafkaAcquisitionProtocol.java index 9238731b48c..ea3cfa86d62 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/acquisition/KafkaAcquisitionProtocol.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/acquisition/KafkaAcquisitionProtocol.java @@ -14,12 +14,15 @@ package org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.connection.KafkaConnection; - public class KafkaAcquisitionProtocol extends AcquisitionProtocol { public String recordTag; public KafkaDataType kafkaDataType; public String connection; + @Override + public boolean isKafkaAcquisitionProtocol() + { + return true; + } }