Skip to content

Commit

Permalink
Add more compile time checks for mastery components (#2436)
Browse files Browse the repository at this point in the history
  • Loading branch information
Adeoye Oluwatobi authored Nov 6, 2023
1 parent 1833643 commit d08df61
Show file tree
Hide file tree
Showing 7 changed files with 502 additions and 7 deletions.
12 changes: 12 additions & 0 deletions ...egend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperAcquisitionBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.DESDecryption;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.Decryption;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.FileAcquisitionProtocol;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.FileType;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.KafkaAcquisitionProtocol;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.LegendServiceAcquisitionProtocol;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.PGPDecryption;
Expand All @@ -42,6 +43,7 @@
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement;

import static java.lang.String.format;
import static org.apache.commons.lang3.StringUtils.isEmpty;

public class HelperAcquisitionBuilder
{
Expand Down Expand Up @@ -75,6 +77,7 @@ public static Root_meta_pure_mastery_metamodel_acquisition_AcquisitionProtocol b

public static Root_meta_pure_mastery_metamodel_acquisition_FileAcquisitionProtocol buildFileAcquisitionProtocol(FileAcquisitionProtocol acquisitionProtocol, CompileContext context)
{
validateFileAcquisitionProtocol(acquisitionProtocol);
Root_meta_pure_mastery_metamodel_connection_FileConnection fileConnection;
PackageableElement packageableElement = context.resolvePackageableElement(acquisitionProtocol.connection, acquisitionProtocol.sourceInformation);
if (packageableElement instanceof Root_meta_pure_mastery_metamodel_connection_FileConnection)
Expand All @@ -98,6 +101,15 @@ public static Root_meta_pure_mastery_metamodel_acquisition_FileAcquisitionProtoc
._decryption(acquisitionProtocol.decryption == null ? null : buildDecryption(acquisitionProtocol.decryption, context));
}

private static void validateFileAcquisitionProtocol(FileAcquisitionProtocol fileAcquisitionProtocol)
{
if (fileAcquisitionProtocol.fileType == FileType.JSON && isEmpty(fileAcquisitionProtocol.recordsKey))
{
throw new EngineException("'recordsKey' must be specified when file type is JSON", fileAcquisitionProtocol.sourceInformation, EngineErrorType.COMPILATION);
}
}


public static Root_meta_pure_mastery_metamodel_acquisition_file_Decryption buildDecryption(Decryption decryption, CompileContext context)
{
if (decryption instanceof PGPDecryption)
Expand Down
53 changes: 50 additions & 3 deletions ...e/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.MasterRecordDefinition;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.Profile;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.RecordService;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.RecordSource;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.RecordSourceVisitor;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.AcquisitionProtocol;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.acquisition.KafkaAcquisitionProtocol;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.authorization.Authorization;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.identity.CollectionEquality;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.identity.IdentityResolution;
Expand All @@ -50,8 +52,12 @@
import java.util.*;
import java.util.stream.Collectors;

import static com.google.common.collect.Iterables.isEmpty;
import static com.google.common.collect.Sets.newHashSet;
import static java.lang.String.format;
import static java.util.Objects.nonNull;
import static org.apache.commons.lang3.BooleanUtils.isFalse;
import static org.apache.commons.lang3.BooleanUtils.isTrue;

public class HelperMasterRecordDefinitionBuilder
{
Expand Down Expand Up @@ -167,6 +173,7 @@ public Root_meta_pure_mastery_metamodel_precedence_PrecedenceRule visit(Preceden
}
else if (precedenceRule instanceof DeleteRule)
{
validateNoDataProviderScope((DeleteRule) precedenceRule);
purePrecedenceRule = new Root_meta_pure_mastery_metamodel_precedence_DeleteRule_Impl("");
}
else if (precedenceRule instanceof CreateRule)
Expand All @@ -175,7 +182,7 @@ else if (precedenceRule instanceof CreateRule)
}
else if (precedenceRule instanceof ConditionalRule)
{
purePrecedenceRule = visitConditionalRule(precedenceRule);
purePrecedenceRule = visitConditionalRule((ConditionalRule) precedenceRule);
}
else
{
Expand Down Expand Up @@ -218,10 +225,10 @@ private Root_meta_pure_mastery_metamodel_precedence_SourcePrecedenceRule visitSo
return pureSourcePrecedenceRule;
}

private Root_meta_pure_mastery_metamodel_precedence_ConditionalRule visitConditionalRule(PrecedenceRule precedenceRule)
private Root_meta_pure_mastery_metamodel_precedence_ConditionalRule visitConditionalRule(ConditionalRule conditionalRule)
{
validateNoScopeSet(conditionalRule);
Root_meta_pure_mastery_metamodel_precedence_ConditionalRule pureConditionalRule = new Root_meta_pure_mastery_metamodel_precedence_ConditionalRule_Impl("");
ConditionalRule conditionalRule = (ConditionalRule) precedenceRule;
validatePredicateInput(conditionalRule.predicate);
pureConditionalRule._predicate(HelperValueSpecificationBuilder.buildLambda(conditionalRule.predicate, context));
return pureConditionalRule;
Expand All @@ -234,6 +241,26 @@ private void validatePredicateInput(Lambda predicate)
validateInputMultiplicity(predicate);
}

private void validateNoScopeSet(ConditionalRule conditionalRule)
{
if (!isEmpty(conditionalRule.scopes))
{
throw new EngineException(
"ConditionalRule with ruleScope is currently unsupported", conditionalRule.sourceInformation,
EngineErrorType.COMPILATION);
}
}

private void validateNoDataProviderScope(DeleteRule deleteRule)
{
if (!isEmpty(deleteRule.scopes) && deleteRule.scopes.stream().anyMatch(scope -> scope instanceof DataProviderTypeScope))
{
throw new EngineException(
"DataProviderTypeScope is not allowed on DeleteRule", deleteRule.sourceInformation,
EngineErrorType.COMPILATION);
}
}

private void validateInputVariableNames(Lambda predicate)
{
Set<String> actualNames = predicate.parameters.stream().map(variable -> variable.name).collect(Collectors.toSet());
Expand Down Expand Up @@ -359,6 +386,7 @@ public RecordSourceBuilder(CompileContext context)
@Override
public Root_meta_pure_mastery_metamodel_RecordSource visit(RecordSource protocolSource)
{
validateRecordSource(protocolSource);
List<IMasteryCompilerExtension> extensions = IMasteryCompilerExtension.getExtensions();
List<Function2<Authorization, CompileContext, Root_meta_pure_mastery_metamodel_authorization_Authorization>> processors = ListIterate.flatCollect(extensions, IMasteryCompilerExtension::getExtraAuthorizationProcessors);
List<Function2<Trigger, CompileContext, Root_meta_pure_mastery_metamodel_trigger_Trigger>> triggerProcessors = ListIterate.flatCollect(extensions, IMasteryCompilerExtension::getExtraTriggerProcessors);
Expand Down Expand Up @@ -415,6 +443,25 @@ private static RichIterable<Root_meta_pure_mastery_metamodel_RecordSourceDepende
new Root_meta_pure_mastery_metamodel_RecordSourceDependency_Impl("")._dependentRecordSourceId(dependency.dependentRecordSourceId)
);
}

private static void validateRecordSource(RecordSource recordSource)
{
if (recordSource.id.length() > 31)
{
throw new EngineException(format("Invalid record source id '%s'; id must not be longer than 31 characters.", recordSource.id), recordSource.sourceInformation, EngineErrorType.COMPILATION);
}

boolean kafkaSource = nonNull(recordSource.recordService) && nonNull(recordSource.recordService.acquisitionProtocol) && recordSource.recordService.acquisitionProtocol.isKafkaAcquisitionProtocol();

if (isTrue(recordSource.sequentialData) && kafkaSource && nonNull(recordSource.runProfile) && recordSource.runProfile != Profile.ExtraSmall)
{
throw new EngineException("'runProfile' can only be set to ExtraSmall for Delta kafka sources", recordSource.sourceInformation, EngineErrorType.COMPILATION);
}
if (kafkaSource && nonNull(recordSource.runProfile) && recordSource.runProfile != Profile.Small)
{
throw new EngineException("'runProfile' can only be set to Small for Full Universe kafka sources", recordSource.sourceInformation, EngineErrorType.COMPILATION);
}
}
}

private static Root_meta_pure_mastery_metamodel_DataProvider getAndValidateDataProvider(String path, SourceInformation sourceInformation, CompileContext context)
Expand Down
34 changes: 34 additions & 0 deletions ...os/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperTriggerBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,19 @@

import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.CompileContext;
import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.trigger.CronTrigger;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.trigger.Frequency;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.trigger.ManualTrigger;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mastery.trigger.Trigger;
import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
import org.finos.legend.pure.generated.Root_meta_pure_mastery_metamodel_trigger_CronTrigger;
import org.finos.legend.pure.generated.Root_meta_pure_mastery_metamodel_trigger_CronTrigger_Impl;
import org.finos.legend.pure.generated.Root_meta_pure_mastery_metamodel_trigger_ManualTrigger_Impl;
import org.finos.legend.pure.generated.Root_meta_pure_mastery_metamodel_trigger_Trigger;

import static org.eclipse.collections.impl.utility.Iterate.isEmpty;

public class HelperTriggerBuilder
{

Expand All @@ -45,6 +50,7 @@ public static Root_meta_pure_mastery_metamodel_trigger_Trigger buildTrigger(Trig

private static Root_meta_pure_mastery_metamodel_trigger_CronTrigger buildCronTrigger(CronTrigger cronTrigger, CompileContext context)
{
validateCronTrigger(cronTrigger);
return new Root_meta_pure_mastery_metamodel_trigger_CronTrigger_Impl("")
._minute(cronTrigger.minute)
._hour(cronTrigger.hour)
Expand All @@ -55,4 +61,32 @@ private static Root_meta_pure_mastery_metamodel_trigger_CronTrigger buildCronTri
._month(cronTrigger.year == null ? null : context.resolveEnumValue("meta::pure::mastery::metamodel::trigger::Month", cronTrigger.month.name()))
._days(ListIterate.collect(cronTrigger.days, day -> context.resolveEnumValue("meta::pure::mastery::metamodel::trigger::Day", day.name())));
}

private static void validateCronTrigger(CronTrigger cronTrigger)
{
if ((cronTrigger.frequency == Frequency.Intraday || cronTrigger.frequency == Frequency.Daily) && isEmpty(cronTrigger.days))
{
throw new EngineException("'days' must not be empty when trigger frequency is Daily or Intraday", cronTrigger.sourceInformation, EngineErrorType.COMPILATION);
}

if (cronTrigger.frequency == Frequency.Weekly && cronTrigger.days.size() != 1)
{
throw new EngineException("'days' specified must be exactly one when trigger frequency is Weekly", cronTrigger.sourceInformation, EngineErrorType.COMPILATION);
}

if (!isInHourRange(cronTrigger.hour) || !isInMinuteRange(cronTrigger.minute))
{
throw new EngineException("'hour' must be a number between 0 and 23 (both inclusive), and 'minute' must be a number between 0 and 59 (both inclusive)", cronTrigger.sourceInformation, EngineErrorType.COMPILATION);
}
}

private static boolean isInHourRange(Integer hour)
{
return 0 <= hour && hour < 24;
}

private static boolean isInMinuteRange(Integer minute)
{
return 0 <= minute && minute < 60;
}
}
4 changes: 2 additions & 2 deletions .../legend/engine/language/pure/dsl/mastery/grammar/from/trigger/TriggerParseTreeWalker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,15 +93,15 @@ private Trigger visitCronTrigger(TriggerParserGrammar.CronTriggerContext ctx)


// frequency
TriggerParserGrammar.FrequencyContext frequencyContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.frequency(), "frequency", sourceInformation);
TriggerParserGrammar.FrequencyContext frequencyContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.frequency(), "frequency", sourceInformation);
if (frequencyContext != null)
{
String frequencyString = frequencyContext.frequencyValue().getText();
cronTrigger.frequency = Frequency.valueOf(frequencyString);
}

// days
TriggerParserGrammar.DaysContext daysContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.days(), "days", sourceInformation);
TriggerParserGrammar.DaysContext daysContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.days(), "days", sourceInformation);
if (daysContext != null)
{
cronTrigger.days = ListIterate.collect(daysContext.dayValue(), this::visitRunDay);
Expand Down
Loading

0 comments on commit d08df61

Please sign in to comment.