Skip to content

Commit

Permalink
Dataspace test data can be used in Service Test Runners (#2635)
Browse files Browse the repository at this point in the history
  • Loading branch information
@haroonsherjan
haroonsherjan authored Feb 16, 2024
1 parent 6c26c18 commit b5ff5f2
Show file tree
Hide file tree
Showing 18 changed files with 187 additions and 48 deletions.
24 changes: 21 additions & 3 deletions ...ava/org/finos/legend/engine/language/pure/compiler/toPureGraph/CoreCompilerExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,20 @@

package org.finos.legend.engine.language.pure.compiler.toPureGraph;

import org.eclipse.collections.api.block.function.Function2;
import org.eclipse.collections.api.block.function.Function3;
import org.eclipse.collections.impl.factory.Maps;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.data.core.EmbeddedDataCompilerHelper;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.data.core.EmbeddedDataCompilerExtension;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.Processor;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.handlers.IncludedMappingHandler;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.handlers.MappingIncludedMappingHandler;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.test.assertion.core.TestAssertionCompilerHelper;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.data.DataElementReference;
import org.finos.legend.engine.protocol.pure.v1.model.data.EmbeddedData;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.data.DataElement;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.MappingIncludeMapping;
import org.finos.legend.engine.protocol.pure.v1.model.test.assertion.TestAssertion;
import org.finos.legend.pure.generated.Root_meta_pure_data_EmbeddedData;
Expand All @@ -32,7 +37,7 @@
import java.util.List;
import java.util.Map;

public class CoreCompilerExtension implements CompilerExtension
public class CoreCompilerExtension implements CompilerExtension, EmbeddedDataCompilerExtension
{
@Override
public Iterable<? extends Processor<?>> getExtraProcessors()
Expand All @@ -43,7 +48,7 @@ public Iterable<? extends Processor<?>> getExtraProcessors()
@Override
public List<Function3<EmbeddedData, CompileContext, ProcessingContext, Root_meta_pure_data_EmbeddedData>> getExtraEmbeddedDataProcessors()
{
return Collections.singletonList(EmbeddedDataCompilerHelper::compileCoreEmbeddedDataTypes);
return Collections.singletonList(EmbeddedDataCompilerExtension::compileCoreEmbeddedDataTypes);
}

@Override
Expand All @@ -60,6 +65,19 @@ public Map<String, IncludedMappingHandler> getExtraIncludedMappingHandlers()
);
}

@Override
public Iterable<? extends Function2<DataElementReference, PureModelContextData, List<EmbeddedData>>> getExtraDataElementReferencePMCDTraversers()
{
return Collections.singletonList(CoreCompilerExtension::getDataFromDataReferencePMCD);
}

private static List<EmbeddedData> getDataFromDataReferencePMCD(DataElementReference dataElementReference, PureModelContextData pureModelContextData)
{
return ListIterate
.select(pureModelContextData.getElementsOfType(DataElement.class), e -> dataElementReference.dataElement.path.equals(e.getPath()))
.collect(d -> d.data);
}

@Override
public CompilerExtension build()
{
Expand Down
58 changes: 37 additions & 21 deletions ...data/core/EmbeddedDataCompilerHelper.java → ...a/core/EmbeddedDataCompilerExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,43 +14,33 @@

package org.finos.legend.engine.language.pure.compiler.toPureGraph.data.core;

import org.eclipse.collections.api.block.function.Function2;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.CompileContext;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.ProcessingContext;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.SourceInformationHelper;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.ValueSpecificationBuilder;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtensions;
import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType;
import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementType;
import org.finos.legend.engine.protocol.pure.v1.model.data.DataElementReference;
import org.finos.legend.engine.protocol.pure.v1.model.data.EmbeddedData;
import org.finos.legend.engine.protocol.pure.v1.model.data.ExternalFormatData;
import org.finos.legend.engine.protocol.pure.v1.model.data.ModelEmbeddedTestData;
import org.finos.legend.engine.protocol.pure.v1.model.data.ModelInstanceTestData;
import org.finos.legend.engine.protocol.pure.v1.model.data.ModelStoreData;
import org.finos.legend.engine.protocol.pure.v1.model.data.ModelTestData;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.data.*;
import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.PackageableElementPtr;
import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
import org.finos.legend.pure.generated.Root_meta_external_format_shared_metamodel_data_ExternalFormatData;
import org.finos.legend.pure.generated.Root_meta_external_format_shared_metamodel_data_ExternalFormatData_Impl;
import org.finos.legend.pure.generated.Root_meta_pure_data_DataElement;
import org.finos.legend.pure.generated.Root_meta_pure_data_DataElementReference_Impl;
import org.finos.legend.pure.generated.Root_meta_pure_data_EmbeddedData;
import org.finos.legend.pure.generated.Root_meta_pure_data_ModelEmbeddedData;
import org.finos.legend.pure.generated.Root_meta_pure_data_ModelEmbeddedData_Impl;
import org.finos.legend.pure.generated.Root_meta_pure_data_ModelInstanceData;
import org.finos.legend.pure.generated.Root_meta_pure_data_ModelInstanceData_Impl;
import org.finos.legend.pure.generated.Root_meta_pure_data_ModelStoreData;
import org.finos.legend.pure.generated.Root_meta_pure_data_ModelStoreData_Impl;
import org.finos.legend.pure.generated.*;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Class;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.InstanceValue;
import org.finos.legend.pure.m4.coreinstance.SourceInformation;

import java.util.Collections;
import java.util.List;
import java.util.Objects;

public class EmbeddedDataCompilerHelper
public interface EmbeddedDataCompilerExtension
{
public static Root_meta_pure_data_EmbeddedData compileCoreEmbeddedDataTypes(EmbeddedData embeddedData, CompileContext context, ProcessingContext processingContext)
static Root_meta_pure_data_EmbeddedData compileCoreEmbeddedDataTypes(EmbeddedData embeddedData, CompileContext context, ProcessingContext processingContext)
{
SourceInformation m3SourceInformation = SourceInformationHelper.toM3SourceInformation(embeddedData.sourceInformation);

Expand Down Expand Up @@ -122,10 +112,36 @@ else if (embeddedData instanceof DataElementReference
}
}

private static boolean validatePairForModelStoreData(PackageableElementPtr value, CompileContext context)
static boolean validatePairForModelStoreData(PackageableElementPtr value, CompileContext context)
{
return (context.pureModel.getPackageableElement(value.fullPath) instanceof Root_meta_pure_data_DataElement &&
((Root_meta_pure_data_DataElement) context.pureModel.getPackageableElement(value.fullPath))._data() instanceof Root_meta_external_format_shared_metamodel_data_ExternalFormatData
);
}

default Iterable<? extends Function2<DataElementReference, PureModelContextData, List<EmbeddedData>>> getExtraDataElementReferencePMCDTraversers()
{
return Collections.emptyList();
}

static EmbeddedData getEmbeddedDataFromDataElement(DataElementReference dataElementReference, PureModelContextData pureModelContextData)
{
List<EmbeddedData> dataList = ListIterate
.selectInstancesOf(CompilerExtensions.fromAvailableExtensions().getExtensions(), EmbeddedDataCompilerExtension.class)
.flatCollect(EmbeddedDataCompilerExtension::getExtraDataElementReferencePMCDTraversers)
.flatCollect(f -> f.apply(dataElementReference, pureModelContextData))
.select(Objects::nonNull);
if (dataList.size() > 1)
{
throw new EngineException("More than one data element found at the address " + dataElementReference.dataElement.path, dataElementReference.sourceInformation, EngineErrorType.COMPILATION);
}
else if (dataList.isEmpty())
{
throw new EngineException("No data element found at the address " + dataElementReference.dataElement.path, dataElementReference.sourceInformation, EngineErrorType.COMPILATION);
}
else
{
return dataList.get(0);
}
}
}
2 changes: 1 addition & 1 deletion ...finos/legend/engine/language/pure/grammar/from/extension/PureGrammarParserExtensions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import org.finos.legend.engine.language.pure.grammar.from.extension.data.EmbeddedDataParser;
import org.finos.legend.engine.language.pure.grammar.from.extension.test.assertion.TestAssertionParser;
import org.finos.legend.engine.language.pure.grammar.from.mapping.MappingIncludeParser;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.MappingInclude;

import java.util.List;
import java.util.ServiceLoader;
Expand Down Expand Up @@ -83,6 +82,7 @@ public MappingIncludeParser getExtraMappingIncludeParser(String type)
return this.mappingIncludeParsers.get(type);
}


public MappingTestInputDataParser getExtraMappingTestInputDataParser(String type)
{
return this.mappingTestInputDataParsers.get(type);
Expand Down
4 changes: 2 additions & 2 deletions .../finos/legend/engine/language/pure/grammar/test/roundtrip/TestDomainGrammarRoundtrip.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,10 +553,10 @@ public void testFunctionTest()
"{\n" +
" MySuite2\n" +
" (\n" +
" ModelStore: (JSON) '[{\"Employees/First Name\":\"John;\\'\\\\\"\",\"Employees/Last Name\":\"Doe\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Nicole\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Time\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Apple\"}]\\n';\n" +
" ModelStore: (JSON) '[{\"Employees/First Name\":\"John\",\"Employees/Last Name\":\"Doe\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Nicole\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Time\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Apple\"}]\\n';\n" +
" store::MyStore: testing::MyReference;\n" +
" myTest | Hello('John') => (XML) 'Hello John!';\n" +
" myTest | Hello('Nicole') => (JSON) '[{\"Employees/First Name\":\"John;\\'\\\\\"\",\"Employees/Last Name\":\"Doe\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Nicole\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Time\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Apple\"}]\\n';\n" +
" myTest | Hello('Nicole') => (JSON) '[{\"Employees/First Name\":\"John\",\"Employees/Last Name\":\"Doe\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Nicole\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Time\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Apple\"}]\\n';\n" +
" )\n" +
"}\n");

Expand Down
2 changes: 1 addition & 1 deletion ...able/src/main/java/org/finos/legend/engine/testable/connection/TestConnectionBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public Pair<Connection, List<Closeable>> visit(Connection connection)
{
String errorMessage = (embeddedData.isEmpty())
? "No test data provided for connection type '" + connection.getClass().getSimpleName() + "'. Either you need to provide test data for the connection type or connection type is not supported."
: "Unsupported test data type '" + embeddedData.getClass().getSimpleName() + "' with connection type '" + connection.getClass().getSimpleName() + '"';
: "Unsupported test data type '" + String.join(",",ListIterate.collect(embeddedData, data -> data.getClass().getSimpleName())) + "' with connection type '" + connection.getClass().getSimpleName() + '"';
return new UnsupportedOperationException(errorMessage);
});
}
Expand Down
21 changes: 20 additions & 1 deletion ...rg/finos/legend/engine/language/pure/compiler/toPureGraph/DataSpaceCompilerExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,24 @@

package org.finos.legend.engine.language.pure.compiler.toPureGraph;

import org.eclipse.collections.api.block.function.Function2;
import org.eclipse.collections.api.block.function.Function3;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.factory.Sets;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.set.MutableSet;
import org.eclipse.collections.impl.utility.Iterate;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.data.EmbeddedDataFirstPassBuilder;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.data.core.EmbeddedDataCompilerExtension;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.Processor;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.handlers.IncludedMappingHandler;
import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType;
import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementType;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.data.DataElementReference;
import org.finos.legend.engine.protocol.pure.v1.model.data.EmbeddedData;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.*;
Expand All @@ -41,7 +45,7 @@

import java.util.*;

public class DataSpaceCompilerExtension implements CompilerExtension
public class DataSpaceCompilerExtension implements CompilerExtension, EmbeddedDataCompilerExtension
{
static final MutableMap<String, Root_meta_pure_metamodel_dataSpace_DataSpace> dataSpacesIndex = Maps.mutable.empty();

Expand Down Expand Up @@ -208,4 +212,19 @@ private Root_meta_pure_data_EmbeddedData compileDataspaceDataElementReference(Em
}
return null;
}


@Override
public Iterable<? extends Function2<DataElementReference, PureModelContextData, List<EmbeddedData>>> getExtraDataElementReferencePMCDTraversers()
{
return org.eclipse.collections.api.factory.Lists.immutable.with(DataSpaceCompilerExtension::getDataFromDataReferencePMCD);
}

private static List<EmbeddedData> getDataFromDataReferencePMCD(DataElementReference dataElementReference, PureModelContextData pureModelContextData)
{
return ListIterate
.select(pureModelContextData.getElementsOfType(DataSpace.class), e -> dataElementReference.dataElement.path.equals(e.getPath()))
.collect(d -> Iterate.detect(d.executionContexts, e -> e.name.equals(d.defaultExecutionContext)).testData)
.collect(d -> EmbeddedDataCompilerExtension.getEmbeddedDataFromDataElement(d, pureModelContextData));
}
}
9 changes: 9 additions & 0 deletions ...ain/java/org/finos/legend/engine/language/pure/grammar/from/DataSpaceParserExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,30 @@
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.eclipse.collections.api.block.function.Function2;
import org.eclipse.collections.impl.factory.Lists;
import org.eclipse.collections.impl.utility.Iterate;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.pure.grammar.from.antlr4.DataSpaceLexerGrammar;
import org.finos.legend.engine.language.pure.grammar.from.antlr4.DataSpaceParserGrammar;
import org.finos.legend.engine.language.pure.grammar.from.antlr4.mapping.MappingParserGrammar;
import org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension;
import org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtensions;
import org.finos.legend.engine.language.pure.grammar.from.extension.SectionParser;
import org.finos.legend.engine.language.pure.grammar.from.extension.data.EmbeddedDataParser;
import org.finos.legend.engine.language.pure.grammar.from.mapping.MappingIncludeParser;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.data.DataElementReference;
import org.finos.legend.engine.protocol.pure.v1.model.data.EmbeddedData;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpace;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.MappingIncludeDataSpace;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.MappingInclude;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.DefaultCodeSection;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.Section;

import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;

public class DataSpaceParserExtension implements PureGrammarParserExtension
Expand Down
2 changes: 1 addition & 1 deletion ...finos/legend/engine/plan/execution/stores/relational/test/HelperRelationalCSVBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public HelperRelationalCSVBuilder(RelationalCSVData relationalData)

public String build()
{
return ListIterate.collect(this.relationalData.tables, this::generateTableCSV).makeString("----\n");
return ListIterate.collect(this.relationalData.tables, this::generateTableCSV).collect(table -> table.endsWith("\n") ? table : table + "\n").makeString("----\n");
}

private String generateTableCSV(RelationalCSVTable table)
Expand Down
4 changes: 2 additions & 2 deletions .../legend/engine/language/pure/grammar/test/TestRelationalEmbeddedDataGrammarRoundtrip.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public void testRelationalEmbeddedData()
" #{\n" +
" default.PersonTable:\n" +
" 'id,firm_id,firstName,lastName\\n'+\n" +
" '1,1,John;\\'\",Doe\\n'+\n" +
" '1,1,John,Doe\\n'+\n" +
" '2,1,Nicole,Smith\\n'+\n" +
" '3,2,Time,Smith\\n';\n" +
"\n" +
Expand All @@ -91,7 +91,7 @@ public void testRelationalEmbeddedData()
" ExternalFormat\n" +
" #{\n" +
" contentType: 'application/json';\n" +
" data: '[{\"Employees/First Name\":\"John;\\'\\\\\"\",\"Employees/Last Name\":\"Doe\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Nicole\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Time\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Apple\"}]\\n';\n" +
" data: '[{\"Employees/First Name\":\"John\",\"Employees/Last Name\":\"Doe\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Nicole\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Finos\"},{\"Employees/First Name\":\"Time\",\"Employees/Last Name\":\"Smith\",\"Legal Name\":\"Apple\"}]\\n';\n" +
" }#;\n" +
" }#\n" +
" ]\n" +
Expand Down
Loading

0 comments on commit b5ff5f2

Please sign in to comment.