Skip to content

Commit

Permalink
add more function test runner tests and build test data from multi st…
Browse files Browse the repository at this point in the history 
…ore (#2560)
  • Loading branch information
@MauricioUyaguari
MauricioUyaguari authored Jan 22, 2024
1 parent fce4554 commit 715a473
Show file tree
Hide file tree
Showing 8 changed files with 288 additions and 37 deletions.
6 changes: 3 additions & 3 deletions ...egend/engine/language/pure/compiler/toPureGraph/test/ModelStoreTestConnectionFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public class ModelStoreTestConnectionFactory implements ConnectionFactoryExtensi
private static XmlModelConnection xmlModelConnection = new XmlModelConnection();


private Pair<Connection, List<Closeable>> resolveExternalFormatData(ExternalFormatData externalFormatData, String _class)
public Pair<Connection, List<Closeable>> buildCloseableConnectionFromExternalFormat(ExternalFormatData externalFormatData, String _class)
{
Closeable closeable = new Closeable()
{
Expand Down Expand Up @@ -98,7 +98,7 @@ private Optional<Pair<Connection, List<Closeable>>> buildModelStoreConnectionsFo
EmbeddedData resolvedEmbeddedData = EmbeddedDataHelper.resolveDataElement(dataElements, modelEmbeddedData.data);
if (resolvedEmbeddedData instanceof ExternalFormatData)
{
return Optional.of(resolveExternalFormatData((ExternalFormatData) resolvedEmbeddedData, _class));
return Optional.of(buildCloseableConnectionFromExternalFormat((ExternalFormatData) resolvedEmbeddedData, _class));
}
else
{
Expand All @@ -114,7 +114,7 @@ else if (data instanceof ModelInstanceTestData)
EmbeddedData testDataElement = EmbeddedDataHelper.findDataElement(dataElements, ((PackageableElementPtr) vs).fullPath).data;
if (testDataElement instanceof ExternalFormatData)
{
return Optional.of(resolveExternalFormatData((ExternalFormatData) testDataElement, _class));
return Optional.of(buildCloseableConnectionFromExternalFormat((ExternalFormatData) testDataElement, _class));
}
else
{
Expand Down
10 changes: 10 additions & 0 deletions ...n/java/org/finos/legend/engine/protocol/pure/v1/extension/ConnectionFactoryExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,5 +61,15 @@ default Optional<Pair<Connection, List<Closeable>>> tryBuildTestConnectionsForSt
return Optional.empty();
}

default Optional<Pair<Connection, List<Closeable>>> tryBuildConnectionForStoreData(Map<String, DataElement> dataElements, Map<Store, EmbeddedData> storeTestData)
{
if (storeTestData.size() == 1)
{
Store store = storeTestData.keySet().stream().findFirst().get();
EmbeddedData embeddedData = storeTestData.values().stream().findFirst().get();
return tryBuildTestConnectionsForStore(dataElements, store, embeddedData);
}
return Optional.empty();
}

}
5 changes: 5 additions & 0 deletions legend-engine-core/legend-engine-core-test/legend-engine-test-runner-function/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,11 @@
<artifactId>legend-engine-xt-serviceStore-javaPlatformBinding-pure</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-executionPlan-execution-store-inMemory</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-configuration</artifactId>
Expand Down
96 changes: 74 additions & 22 deletions ...src/main/java/org/finos/legend/engine/testable/function/extension/FunctionTestRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
import org.eclipse.collections.impl.tuple.Tuples;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.ConnectionFirstPassBuilder;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.HelperModelBuilder;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.test.ModelStoreTestConnectionFactory;
import org.finos.legend.engine.plan.execution.PlanExecutor;
import org.finos.legend.engine.plan.execution.planHelper.PrimitiveValueSpecificationToObjectVisitor;
import org.finos.legend.engine.plan.execution.result.Result;
Expand All @@ -36,6 +38,7 @@
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.data.EmbeddedData;
import org.finos.legend.engine.protocol.pure.v1.model.data.EmbeddedDataHelper;
import org.finos.legend.engine.protocol.pure.v1.model.data.ExternalFormatData;
import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.Connection;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function;
Expand All @@ -56,6 +59,9 @@
import org.finos.legend.engine.testable.extension.TestRunner;
import org.finos.legend.pure.generated.Root_meta_core_runtime_Connection;
import org.finos.legend.pure.generated.Root_meta_core_runtime_ConnectionStore;
import org.finos.legend.pure.generated.Root_meta_external_store_model_JsonModelConnection;
import org.finos.legend.pure.generated.Root_meta_external_store_model_PureModelConnection;
import org.finos.legend.pure.generated.Root_meta_external_store_model_XmlModelConnection;
import org.finos.legend.pure.generated.Root_meta_legend_function_metamodel_FunctionTestSuite;
import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;
import org.finos.legend.pure.generated.Root_meta_pure_test_AtomicTest;
Expand Down Expand Up @@ -83,7 +89,8 @@ public class FunctionTestRunner implements TestRunner
private final MutableList<PlanGeneratorExtension> extensions;
private final PlanExecutor executor;
private final String pureVersion;
private final MutableList<ConnectionFactoryExtension> factories = org.eclipse.collections.api.factory.Lists.mutable.withAll(ServiceLoader.load(ConnectionFactoryExtension.class));

private final MutableList<ConnectionFactoryExtension> connectionBuilders = org.eclipse.collections.api.factory.Lists.mutable.withAll(ServiceLoader.load(ConnectionFactoryExtension.class));
private List<Closeable> closeables = Lists.mutable.empty();
private List<Pair<Root_meta_core_runtime_ConnectionStore, Root_meta_core_runtime_Connection>> storeConnectionsPairs = Lists.mutable.empty();

Expand Down Expand Up @@ -201,35 +208,80 @@ private void setup(FunctionTestRunnerContext context)
{
return;
}
runtime._connectionStores().forEach(connectionStore ->
Map<Root_meta_core_runtime_Connection, List<org.finos.legend.pure.generated.Root_meta_core_runtime_ConnectionStore>> connectionMap = Maps.mutable.empty();
runtime._connectionStores().forEach(connectionStores ->
{
List<org.finos.legend.pure.generated.Root_meta_core_runtime_ConnectionStore> stores = connectionMap.get(connectionStores._connection());
if (stores == null)
{
stores = Lists.mutable.empty();
connectionMap.putIfAbsent(connectionStores._connection(), stores);
}
stores.add(connectionStores);
});
for (Map.Entry<Root_meta_core_runtime_Connection, List<org.finos.legend.pure.generated.Root_meta_core_runtime_ConnectionStore>> entry : connectionMap.entrySet())
{
// find connections that have been mocked and replace
Object element = connectionStore._element();
if (element instanceof org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store)

Root_meta_core_runtime_Connection key = entry.getKey();
List<org.finos.legend.pure.generated.Root_meta_core_runtime_ConnectionStore> values = entry.getValue();
Map<Store, EmbeddedData> storeTestDataList = Maps.mutable.empty();
// collect test data
entry.getValue().forEach(connectionStore ->
{
org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store store = (org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store) element;
String storePath = getElementFullPath(store, context.getPureModel().getExecutionSupport());
Optional<StoreTestData> optionalStoreTestData = protocolFunctionSuite.testData.stream().filter(pTestData -> pTestData.store.equals(storePath)).findFirst();
if (optionalStoreTestData.isPresent())
Object element = connectionStore._element();
if (element instanceof org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store)
{
StoreTestData storeTestData = optionalStoreTestData.get();
EmbeddedData testData = EmbeddedDataHelper.resolveEmbeddedDataInPMCD(context.getPureModelContextData(), storeTestData.data);
Pair<Connection, List<Closeable>> closeableMockedConnections = this.factories.collect(f -> f.tryBuildTestConnectionsForStore(context.getDataElementIndex(), resolveStore(context.getPureModelContextData(), storePath), testData)).select(Objects::nonNull).select(Optional::isPresent)
.collect(Optional::get).getFirstOptional().orElseThrow(() -> new UnsupportedOperationException("Unsupported store type for: '" + storePath + "' mentioned while running the function tests"));
Connection mockedConnection = closeableMockedConnections.getOne();
Root_meta_core_runtime_Connection mockedCompileConnection = mockedConnection.accept(context.getConnectionVisitor());
// we replace with mocked connection. We set back to original at cleanup
Root_meta_core_runtime_Connection realConnection = connectionStore._connection();
this.storeConnectionsPairs.add(Tuples.pair(connectionStore, realConnection));
connectionStore._connection(mockedCompileConnection);
org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store metamodelStore = (org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store) element;
String storePath = getElementFullPath(metamodelStore, context.getPureModel().getExecutionSupport());
Optional<StoreTestData> optionalStoreTestData = protocolFunctionSuite.testData.stream().filter(pTestData -> pTestData.store.equals(storePath)).findFirst();
if (optionalStoreTestData.isPresent())
{
StoreTestData resolvedStoreTestData = optionalStoreTestData.get();
EmbeddedData testData = EmbeddedDataHelper.resolveEmbeddedDataInPMCD(context.getPureModelContextData(), resolvedStoreTestData.data);
Store store = this.resolveStore(context.getPureModelContextData(), resolvedStoreTestData.store);
storeTestDataList.put(store, testData);
}
}
});
if (!storeTestDataList.isEmpty())
{
Pair<Connection, List<Closeable>> closeableMockedConnections = this.buildTestConnection(context, key, storeTestDataList);
Connection mockedConnection = closeableMockedConnections.getOne();
Root_meta_core_runtime_Connection mockedCompileConnection = mockedConnection.accept(context.getConnectionVisitor());
// we replace with mocked connection. We set back to original at cleanup
for (Root_meta_core_runtime_ConnectionStore value : values)
{
Root_meta_core_runtime_Connection realConnection = value._connection();
this.storeConnectionsPairs.add(Tuples.pair(value, realConnection));
value._connection(mockedCompileConnection);
this.closeables.addAll(closeableMockedConnections.getTwo());
}
else
}
}
}

private Pair<Connection, List<Closeable>> buildTestConnection(FunctionTestRunnerContext context,Root_meta_core_runtime_Connection connection, Map<Store, EmbeddedData> storeEmbeddedDataMap)
{
if (storeEmbeddedDataMap.size() == 1 && connection instanceof Root_meta_external_store_model_PureModelConnection)
{
EmbeddedData embeddedData = storeEmbeddedDataMap.values().stream().findFirst().get();
if (embeddedData instanceof ExternalFormatData)
{
ExternalFormatData externalFormatData = (ExternalFormatData) embeddedData;
if (connection instanceof Root_meta_external_store_model_JsonModelConnection)
{
String _class = HelperModelBuilder.getElementFullPath(((Root_meta_external_store_model_JsonModelConnection) connection)._class(), context.getPureModel().getExecutionSupport());
return new ModelStoreTestConnectionFactory().buildCloseableConnectionFromExternalFormat(externalFormatData, _class);
}
else if (connection instanceof Root_meta_external_store_model_XmlModelConnection)
{
LOGGER.warn("No test data found store + " + storePath + ". When building test data for runtime");
String _class = HelperModelBuilder.getElementFullPath(((Root_meta_external_store_model_XmlModelConnection) connection)._class(), context.getPureModel().getExecutionSupport());
return new ModelStoreTestConnectionFactory().buildCloseableConnectionFromExternalFormat(externalFormatData, _class);
}
}
});
}
return this.connectionBuilders.collect(f -> f.tryBuildConnectionForStoreData(context.getDataElementIndex(), storeEmbeddedDataMap)).select(Objects::nonNull).select(Optional::isPresent)
.collect(Optional::get).getFirstOptional().orElseThrow(() -> new UnsupportedOperationException("Unsupported test data for function test suite: " + context.getTestSuite()._id()));
}

private void tearDown()
Expand Down
Loading

0 comments on commit 715a473

Please sign in to comment.