From a890710d9fceb515b8523dbb79e91c5a91fc9042 Mon Sep 17 00:00:00 2001 From: Mauricio Uyaguari Date: Thu, 20 Jun 2024 13:25:08 -0400 Subject: [PATCH] add test debug and add exection plan to mapping test debug (#2916) --- .../pure/v1/CorePureProtocolExtension.java | 6 + .../pure/v1/model/test/result/TestDebug.java | 32 ++++ .../test/result/TestExecutionPlanDebug.java | 26 +++ .../mapping/extension/MappingTestRunner.java | 156 ++++++++++++++---- .../extension/MappingTestRunnerContext.java | 9 +- .../extension/TestMappingTestRunner.java | 34 +++- .../engine/testable/api/TestableApi.java | 23 +++ .../engine/testable/TestableRunner.java | 48 ++++++ .../engine/testable/extension/TestRunner.java | 12 ++ .../testable/model/DebugTestsResult.java | 25 +++ 10 files changed, 337 insertions(+), 34 deletions(-) create mode 100644 legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/test/result/TestDebug.java create mode 100644 legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/test/result/TestExecutionPlanDebug.java create mode 100644 legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/model/DebugTestsResult.java diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/CorePureProtocolExtension.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/CorePureProtocolExtension.java index 217496297c1..612f3d4ddc4 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/CorePureProtocolExtension.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/CorePureProtocolExtension.java @@ -70,6 +70,8 @@ import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestError; import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestExecuted; import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestResult; +import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestDebug; +import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestExecutionPlanDebug; import java.util.List; import java.util.Map; @@ -130,6 +132,10 @@ public List>>> getExtraProtocolSubTypeInfo .withSubtype(TestError.class, "testError") .withSubtype(TestExecuted.class, "testExecuted") .build(), + // Test Debug + ProtocolSubTypeInfo.newBuilder(TestDebug.class) + .withSubtype(TestExecutionPlanDebug.class, "testExecutionPlanDebug") + .build(), // Assertion Status ProtocolSubTypeInfo.newBuilder(AssertionStatus.class) .withSubtype(AssertPass.class, "assertPass") diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/test/result/TestDebug.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/test/result/TestDebug.java new file mode 100644 index 00000000000..c6251ac9221 --- /dev/null +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/test/result/TestDebug.java @@ -0,0 +1,32 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.pure.v1.model.test.result; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type") +public abstract class TestDebug +{ + @JsonProperty(required = true) + public String testable; + + public String testSuiteId; + + @JsonProperty(required = true) + public String atomicTestId; + + public String error; +} diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/test/result/TestExecutionPlanDebug.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/test/result/TestExecutionPlanDebug.java new file mode 100644 index 00000000000..4d3360c0127 --- /dev/null +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/test/result/TestExecutionPlanDebug.java @@ -0,0 +1,26 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.pure.v1.model.test.result; + +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.ExecutionPlan; + +import java.util.List; + +public class TestExecutionPlanDebug extends TestDebug +{ + public ExecutionPlan executionPlan; + + public List debug; +} diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/main/java/org/finos/legend/engine/testable/mapping/extension/MappingTestRunner.java b/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/main/java/org/finos/legend/engine/testable/mapping/extension/MappingTestRunner.java index a32615bb3e5..e84c3488fd8 100644 --- a/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/main/java/org/finos/legend/engine/testable/mapping/extension/MappingTestRunner.java +++ b/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/main/java/org/finos/legend/engine/testable/mapping/extension/MappingTestRunner.java @@ -25,11 +25,13 @@ import org.finos.legend.engine.plan.execution.result.Result; import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat; import org.finos.legend.engine.plan.generation.PlanGenerator; +import org.finos.legend.engine.plan.generation.PlanWithDebug; import org.finos.legend.engine.plan.generation.extension.PlanGeneratorExtension; import org.finos.legend.engine.plan.platform.PlanPlatform; import org.finos.legend.engine.protocol.pure.v1.extension.ConnectionFactoryExtension; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; -import org.finos.legend.engine.protocol.pure.v1.model.data.*; +import org.finos.legend.engine.protocol.pure.v1.model.data.EmbeddedData; +import org.finos.legend.engine.protocol.pure.v1.model.data.EmbeddedDataHelper; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.Connection; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.mappingTest.MappingTest; @@ -41,6 +43,8 @@ import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestError; import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestExecuted; import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestResult; +import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestDebug; +import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestExecutionPlanDebug; import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; import org.finos.legend.engine.shared.core.operational.Assert; import org.finos.legend.engine.testable.assertion.TestAssertionEvaluator; @@ -82,18 +86,19 @@ public TestResult executeAtomicTest(Root_meta_pure_test_AtomicTest atomicTest, P throw new UnsupportedOperationException("Mapping Test should be executed in context of Mapping Test Suite only"); } + @Override + public TestDebug debugAtomicTest(Root_meta_pure_test_AtomicTest atomicTest, PureModel pureModel, PureModelContextData pmcd) + { + throw new UnsupportedOperationException("Mapping Test should be executed in context of Mapping Test Suite only"); + } + @Override public List executeTestSuite(Root_meta_pure_test_TestSuite testSuite, List atomicTestIds, PureModel pureModel, PureModelContextData pureModelContextData) { List results = Lists.mutable.empty(); - String testablePath = getElementFullPath(this.pureMapping, pureModel.getExecutionSupport()); - Assert.assertTrue(testSuite instanceof Root_meta_pure_mapping_metamodel_MappingTestSuite, () -> "Test Suite in Mapping expected to be of type Mapping Test Suite"); - Root_meta_pure_mapping_metamodel_MappingTestSuite compiledMappingTestSuite = (Root_meta_pure_mapping_metamodel_MappingTestSuite) testSuite; - try + MappingTestRunnerContext context = buildMappingContext(testSuite, pureModel, pureModelContextData); try { - org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping mapping = ListIterate.detect(pureModelContextData.getElementsOfType(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping.class), ele -> ele.getPath().equals(testablePath)); - MappingTestSuite mappingTestSuite = ListIterate.detect(mapping.testSuites, ts -> ts.id.equals(testSuite._id())); - MappingTestRunnerContext context = new MappingTestRunnerContext(compiledMappingTestSuite, mapping, pureModel, pureModelContextData, extensions.flatCollect(PlanGeneratorExtension::getExtraPlanTransformers), new ConnectionFirstPassBuilder(pureModel.getContext()), PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport()))); + MappingTestSuite mappingTestSuite = ListIterate.detect(context.getMapping().testSuites, ts -> ts.id.equals(testSuite._id())); // build plan, executor args List mappingTests = mappingTestSuite.tests.stream().filter(t -> t instanceof MappingTest).map(t -> (MappingTest)t).collect(Collectors.toList()); // running of each test is catchable and put under a test error @@ -102,8 +107,8 @@ public List executeTestSuite(Root_meta_pure_test_TestSuite testSuite if (atomicTestIds.contains(mappingTest.id)) { org.finos.legend.engine.protocol.pure.v1.model.test.result.TestResult testResult = executeMappingTest(mappingTest, context); - testResult.testable = testablePath; - testResult.testSuiteId = compiledMappingTestSuite._id(); + testResult.testable = context.getMapping().getPath(); + testResult.testSuiteId = context.getMetamodelTestSuite()._id(); results.add(testResult); } } @@ -112,7 +117,7 @@ public List executeTestSuite(Root_meta_pure_test_TestSuite testSuite catch (Exception e) { // this is to catch any error for the setup of the test suite. we return test error for each test run - for (Root_meta_pure_test_AtomicTest testedError: compiledMappingTestSuite._tests()) + for (Root_meta_pure_test_AtomicTest testedError: context.getMetamodelTestSuite()._tests()) { if (atomicTestIds.contains(testedError._id()) && results.stream().noneMatch(t -> t.atomicTestId.equals(testedError._id()))) { @@ -126,27 +131,60 @@ public List executeTestSuite(Root_meta_pure_test_TestSuite testSuite return results; } + @Override + public List debugTestSuite(Root_meta_pure_test_TestSuite testSuite, List atomicTestIds, PureModel pureModel, PureModelContextData pureModelContextData) + { + List results = Lists.mutable.empty(); + MappingTestRunnerContext context = buildMappingContext(testSuite, pureModel, pureModelContextData); + try + { + MappingTestSuite mappingTestSuite = ListIterate.detect(context.getMapping().testSuites, ts -> ts.id.equals(testSuite._id())); + // build plan, executor args + List mappingTests = mappingTestSuite.tests.stream().filter(t -> t instanceof MappingTest).map(t -> (MappingTest)t).collect(Collectors.toList()); + // running of each test is catchable and put under a test error + for (MappingTest mappingTest : mappingTests) + { + if (atomicTestIds.contains(mappingTest.id)) + { + org.finos.legend.engine.protocol.pure.v1.model.test.result.TestDebug testResult = debugMappingTest(mappingTest, context); + testResult.testable = context.getMapping().getPath(); + testResult.testSuiteId = context.getMetamodelTestSuite()._id(); + results.add(testResult); + } + } + } + catch (Exception e) + { + // this is to catch any error for the setup of the test suite. we return test error for each test run + for (Root_meta_pure_test_AtomicTest testedError: context.getMetamodelTestSuite()._tests()) + { + if (atomicTestIds.contains(testedError._id()) && results.stream().noneMatch(t -> t.atomicTestId.equals(testedError._id()))) + { + TestExecutionPlanDebug testError = new TestExecutionPlanDebug(); + testError.atomicTestId = testedError._id(); + testError.error = e.toString(); + results.add(testError); + } + } + } + return results; + } + + private MappingTestRunnerContext buildMappingContext(Root_meta_pure_test_TestSuite testSuite, PureModel pureModel, PureModelContextData pureModelContextData) + { + String testablePath = getElementFullPath(this.pureMapping, pureModel.getExecutionSupport()); + Assert.assertTrue(testSuite instanceof Root_meta_pure_mapping_metamodel_MappingTestSuite, () -> "Test Suite in Mapping expected to be of type Mapping Test Suite"); + Root_meta_pure_mapping_metamodel_MappingTestSuite compiledMappingTestSuite = (Root_meta_pure_mapping_metamodel_MappingTestSuite) testSuite; + org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping mapping = ListIterate.detect(pureModelContextData.getElementsOfType(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping.class), ele -> ele.getPath().equals(testablePath)); + return new MappingTestRunnerContext(compiledMappingTestSuite, mapping, pureModel, pureModelContextData, extensions.flatCollect(PlanGeneratorExtension::getExtraPlanTransformers), new ConnectionFirstPassBuilder(pureModel.getContext()), PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport()))); + } + private TestResult executeMappingTest(MappingTest mappingTest, MappingTestRunnerContext context) { List>> connections = Lists.mutable.empty(); try { - Root_meta_core_runtime_Runtime_Impl runtime = new Root_meta_core_runtime_Runtime_Impl(""); - List> connectionInfo = mappingTest.storeTestData.stream().map(testData -> Tuples.pair(testData.store, EmbeddedDataHelper.resolveEmbeddedDataInPMCD(context.getPureModelContextData(), testData.data))).collect(Collectors.toList()); - connections = connectionInfo.stream() - .map(pair -> this.factories.collect(f -> f.tryBuildTestConnectionsForStore(context.getDataElementIndex(), resolveStore(context.getPureModelContextData(), pair.getOne()), pair.getTwo())).select(Objects::nonNull).select(Optional::isPresent) - .collect(Optional::get).getFirstOptional().orElseThrow(() -> new UnsupportedOperationException("Unsupported store type for:'" + pair.getOne() + "' mentioned while running the mapping tests"))).collect(Collectors.toList()); - connections.forEach(connection -> - { - Connection conn = connection.getOne(); - org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store element = getStore(conn.element, conn.elementSourceInformation, context.getPureModel().getContext()); - Root_meta_core_runtime_ConnectionStore connectionStore = - new Root_meta_core_runtime_ConnectionStore_Impl("") - ._connection(conn.accept(context.getConnectionVisitor())) - ._element(element); - runtime._connectionStoresAdd(connectionStore); - }); - handleGenerationOfPlan(connections.stream().map(Pair::getOne).collect(Collectors.toList()), runtime, context); + connections = generateExecutionPlan(mappingTest, context, false); // execute assertion TestAssertion assertion = mappingTest.assertions.get(0); PlanExecutor.ExecuteArgs executeArgs = context.getExecuteBuilder().build(); @@ -169,7 +207,55 @@ private TestResult executeMappingTest(MappingTest mappingTest, MappingTestRunne } } - private void handleGenerationOfPlan(List incomingConnections, Root_meta_core_runtime_Runtime_Impl runtime, MappingTestRunnerContext context) + private TestDebug debugMappingTest(MappingTest mappingTest, MappingTestRunnerContext context) + { + List>> connections = Lists.mutable.empty(); + + try + { + TestExecutionPlanDebug executionPlanDebug = new TestExecutionPlanDebug(); + connections = generateExecutionPlan(mappingTest, context, true); + executionPlanDebug.executionPlan = context.getPlan(); + executionPlanDebug.debug = context.getDebug(); + executionPlanDebug.atomicTestId = mappingTest.id; + return executionPlanDebug; + } + catch (Exception e) + { + TestExecutionPlanDebug executionPlanDebug = new TestExecutionPlanDebug(); + executionPlanDebug.atomicTestId = mappingTest.id; + executionPlanDebug.error = e.toString(); + return executionPlanDebug; + } + finally + { + this.closeConnections(connections); + } + } + + private List>> generateExecutionPlan(MappingTest mappingTest, MappingTestRunnerContext context, boolean debug) + { + + Root_meta_core_runtime_Runtime_Impl runtime = new Root_meta_core_runtime_Runtime_Impl(""); + List> connectionInfo = mappingTest.storeTestData.stream().map(testData -> Tuples.pair(testData.store, EmbeddedDataHelper.resolveEmbeddedDataInPMCD(context.getPureModelContextData(), testData.data))).collect(Collectors.toList()); + List>> connections = connectionInfo.stream() + .map(pair -> this.factories.collect(f -> f.tryBuildTestConnectionsForStore(context.getDataElementIndex(), resolveStore(context.getPureModelContextData(), pair.getOne()), pair.getTwo())).select(Objects::nonNull).select(Optional::isPresent) + .collect(Optional::get).getFirstOptional().orElseThrow(() -> new UnsupportedOperationException("Unsupported store type for:'" + pair.getOne() + "' mentioned while running the mapping tests"))).collect(Collectors.toList()); + connections.forEach(connection -> + { + Connection conn = connection.getOne(); + org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store element = getStore(conn.element, conn.elementSourceInformation, context.getPureModel().getContext()); + Root_meta_core_runtime_ConnectionStore connectionStore = + new Root_meta_core_runtime_ConnectionStore_Impl("") + ._connection(conn.accept(context.getConnectionVisitor())) + ._element(element); + runtime._connectionStoresAdd(connectionStore); + }); + handleGenerationOfPlan(connections.stream().map(Pair::getOne).collect(Collectors.toList()), runtime, context, debug); + return connections; + } + + private void handleGenerationOfPlan(List incomingConnections, Root_meta_core_runtime_Runtime_Impl runtime, MappingTestRunnerContext context, boolean debug) { SingleExecutionPlan executionPlan = context.getPlan(); boolean reusePlan = false; @@ -183,8 +269,18 @@ private void handleGenerationOfPlan(List incomingConnections, Root_m } if (executionPlan == null || !reusePlan) { - executionPlan = PlanGenerator.generateExecutionPlan(context.getMetamodelTestSuite()._query(), pureMapping, runtime, null, context.getPureModel(), this.pureVersion, PlanPlatform.JAVA, null, context.getRouterExtensions(), context.getExecutionPlanTransformers()); - context.withPlan(executionPlan); + List debugger = null; + if (debug) + { + PlanWithDebug plan = PlanGenerator.generateExecutionPlanDebug(context.getMetamodelTestSuite()._query(), pureMapping, runtime, null, context.getPureModel(), this.pureVersion, PlanPlatform.JAVA, null, context.getRouterExtensions(), context.getExecutionPlanTransformers()); + executionPlan = plan.plan; + debugger = Arrays.asList(plan.debug); + } + else + { + executionPlan = PlanGenerator.generateExecutionPlan(context.getMetamodelTestSuite()._query(), pureMapping, runtime, null, context.getPureModel(), this.pureVersion, PlanPlatform.JAVA, null, context.getRouterExtensions(), context.getExecutionPlanTransformers()); + } + context.withPlan(executionPlan, debugger); } // set new connections context.withConnections(incomingConnections); diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/main/java/org/finos/legend/engine/testable/mapping/extension/MappingTestRunnerContext.java b/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/main/java/org/finos/legend/engine/testable/mapping/extension/MappingTestRunnerContext.java index 0b0ca6a94de..3a5f8c31942 100644 --- a/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/main/java/org/finos/legend/engine/testable/mapping/extension/MappingTestRunnerContext.java +++ b/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/main/java/org/finos/legend/engine/testable/mapping/extension/MappingTestRunnerContext.java @@ -45,6 +45,7 @@ class MappingTestRunnerContext private final PlanExecutor.ExecuteArgsBuilder executeBuilder; private final Map dataElementIndex; private SingleExecutionPlan plan; + private List debug; private List connections; public MappingTestRunnerContext(Root_meta_pure_mapping_metamodel_MappingTestSuite metamodelTestSuite, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping mapping, PureModel pureModel, PureModelContextData pureModelContextData, MutableList executionPlanTransformers, @@ -108,9 +109,15 @@ public SingleExecutionPlan getPlan() return plan; } - public void withPlan(SingleExecutionPlan plan) + public List getDebug() + { + return this.debug; + } + + public void withPlan(SingleExecutionPlan plan, List debug) { this.executeBuilder.withPlan(plan); + this.debug = debug; this.plan = plan; } diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/test/java/org/finos/legend/engine/testable/mapping/extension/TestMappingTestRunner.java b/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/test/java/org/finos/legend/engine/testable/mapping/extension/TestMappingTestRunner.java index 828158e889f..dfde4032524 100644 --- a/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/test/java/org/finos/legend/engine/testable/mapping/extension/TestMappingTestRunner.java +++ b/legend-engine-core/legend-engine-core-testable/legend-engine-test-runner-mapping/src/test/java/org/finos/legend/engine/testable/mapping/extension/TestMappingTestRunner.java @@ -22,12 +22,13 @@ import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.test.assertion.status.AssertionStatus; import org.finos.legend.engine.protocol.pure.v1.model.test.assertion.status.EqualToJsonAssertFail; -import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestExecuted; -import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestExecutionStatus; -import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestResult; +import org.finos.legend.engine.protocol.pure.v1.model.test.result.*; import org.finos.legend.engine.shared.core.deployment.DeploymentMode; import org.finos.legend.engine.shared.core.identity.Identity; import org.finos.legend.engine.shared.core.identity.factory.*; +import org.finos.legend.engine.testable.extension.TestRunner; +import org.finos.legend.pure.generated.Root_meta_pure_test_TestSuite; +import org.finos.legend.pure.m3.coreinstance.meta.pure.test.TestAccessor; import org.junit.Assert; import org.junit.Test; @@ -614,6 +615,33 @@ public void testMappingTestSuiteForM2MUsecase() Assert.assertEquals("test1", mappingTestResults.get(0).atomicTestId); } + @Test + public void testDebugMappingTestSuiteForM2MUsecase() + { + MappingTestableRunnerExtension mappingTestableRunnerExtension = new MappingTestableRunnerExtension(); + mappingTestableRunnerExtension.setPureVersion(PureClientVersions.production); + + PureModelContextData modelDataWithReferenceData = PureGrammarParser.newInstance().parseModel(TEST_SUITE_1); + PureModel pureModelWithReferenceData = Compiler.compile(modelDataWithReferenceData, DeploymentMode.TEST, Identity.getAnonymousIdentity().getName()); + org.finos.legend.pure.m3.coreinstance.meta.pure.mapping.Mapping mappingToTest = (org.finos.legend.pure.m3.coreinstance.meta.pure.mapping.Mapping) pureModelWithReferenceData.getPackageableElement("test::modelToModelMapping"); + TestRunner runner = mappingTestableRunnerExtension.getTestRunner(mappingToTest); + List debugLists = mappingToTest._tests().flatCollect(testSuite -> + { + List atomicTestIds = ((Root_meta_pure_test_TestSuite) testSuite)._tests().collect(TestAccessor::_id).toList(); + return runner.debugTestSuite((Root_meta_pure_test_TestSuite) testSuite, atomicTestIds, pureModelWithReferenceData, modelDataWithReferenceData); + }).toList(); + Assert.assertEquals(1, debugLists.size()); + Assert.assertEquals("test::modelToModelMapping", debugLists.get(0).testable); + Assert.assertEquals("testSuite1", debugLists.get(0).testSuiteId); + Assert.assertEquals("test1", debugLists.get(0).atomicTestId); + TestDebug testDebug = debugLists.get(0); + Assert.assertTrue(testDebug instanceof TestExecutionPlanDebug); + TestExecutionPlanDebug testExecutionPlanDebug = (TestExecutionPlanDebug) testDebug; + Assert.assertNotNull(testExecutionPlanDebug.executionPlan); + Assert.assertNull(testExecutionPlanDebug.error); + Assert.assertFalse(testExecutionPlanDebug.debug.isEmpty()); + } + @Test public void testMappingTestSuiteForM2MThreeLevelDeep() { diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-testable-http-api/src/main/java/org/finos/legend/engine/testable/api/TestableApi.java b/legend-engine-core/legend-engine-core-testable/legend-engine-testable-http-api/src/main/java/org/finos/legend/engine/testable/api/TestableApi.java index b80de778237..a8eebadfcb2 100644 --- a/legend-engine-core/legend-engine-core-testable/legend-engine-testable-http-api/src/main/java/org/finos/legend/engine/testable/api/TestableApi.java +++ b/legend-engine-core/legend-engine-core-testable/legend-engine-testable-http-api/src/main/java/org/finos/legend/engine/testable/api/TestableApi.java @@ -35,6 +35,7 @@ import org.finos.legend.engine.shared.core.operational.prometheus.MetricsHandler; import org.finos.legend.engine.shared.core.operational.prometheus.Prometheus; import org.finos.legend.engine.testable.TestableRunner; +import org.finos.legend.engine.testable.model.DebugTestsResult; import org.finos.legend.engine.testable.model.RunTestsInput; import javax.ws.rs.Consumes; @@ -99,4 +100,26 @@ public Response doTests(RunTestsInput input, @ApiParam(hidden = true) @Pac4JProf return ExceptionTool.exceptionManager(e, LoggingEventType.TESTABLE_DO_TESTS_ERROR, identity.getName()); } } + + @POST + @Path("debugTests") + @ApiOperation(value = "Debug testables") + @Consumes({MediaType.APPLICATION_JSON, InflateInterceptor.APPLICATION_ZLIB}) + public Response debugTests(RunTestsInput input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager profileManager) + { + MutableList profiles = ProfileManagerHelper.extractProfiles(profileManager); + Identity identity = Identity.makeIdentity(profiles); + try + { + LOGGER.info(new LogInfo(identity.getName(), LoggingEventType.TESTABLE_DO_TESTS_START, "").toString()); + Pair modelAndData = this.modelManager.loadModelAndData(input.model, input.model instanceof PureModelContextPointer ? ((PureModelContextPointer) input.model).serializer.version : null, identity, null); + DebugTestsResult runTestsResult = testableRunner.debugTests(input.testables, modelAndData.getTwo(), modelAndData.getOne()); + LOGGER.info(new LogInfo(identity.getName(), LoggingEventType.TESTABLE_DO_TESTS_STOP, "").toString()); + return ManageConstantResult.manageResult(identity.getName(), runTestsResult, objectMapper); + } + catch (Exception e) + { + return ExceptionTool.exceptionManager(e, LoggingEventType.TESTABLE_DO_TESTS_ERROR, identity.getName()); + } + } } diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/TestableRunner.java b/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/TestableRunner.java index dff771c6ee0..79138a42de9 100644 --- a/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/TestableRunner.java +++ b/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/TestableRunner.java @@ -17,8 +17,11 @@ import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestDebug; +import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestResult; import org.finos.legend.engine.testable.extension.TestRunner; import org.finos.legend.engine.testable.extension.TestableRunnerExtensionLoader; +import org.finos.legend.engine.testable.model.DebugTestsResult; import org.finos.legend.engine.testable.model.UniqueTestId; import org.finos.legend.engine.testable.model.RunTestsResult; import org.finos.legend.engine.testable.model.RunTestsTestableInput; @@ -72,4 +75,49 @@ public RunTestsResult doTests(List runTestsTestableInputs } return runTestsResult; } + + + public DebugTestsResult debugTests(List runTestsTestableInputs, PureModel pureModel, PureModelContextData data) + { + DebugTestsResult debugTestsResult = new DebugTestsResult(); + for (RunTestsTestableInput testableInput : runTestsTestableInputs) + { + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = pureModel.getPackageableElement(testableInput.testable); + if (!(packageableElement instanceof Testable)) + { + throw new UnsupportedOperationException("Element '" + testableInput.testable + "' is not a testable element"); + } + Testable testable = (Testable) packageableElement; + List testIds = testableInput.unitTestIds; + List testIdStrings = ListIterate.collect(testIds, id -> id.atomicTestId); + + TestRunner testRunner = TestableRunnerExtensionLoader.forTestable(testable); + for (Test test : testable._tests()) + { + // We run all testIds if no `unitTestIds` are provided + if ((test instanceof Root_meta_pure_test_AtomicTest) && (testIds.isEmpty() || testIdStrings.contains(test._id()))) + { + TestDebug testDebug = testRunner.debugAtomicTest((Root_meta_pure_test_AtomicTest) test, pureModel, data); + if (testDebug != null) + { + debugTestsResult.results.add(testDebug); + } + } + + if (test instanceof Root_meta_pure_test_TestSuite) + { + List testIdsForSuite = ListIterate.collectIf(testIds, testId -> test._id().equals(testId.testSuiteId), testId -> testId.atomicTestId); + if (testIds.isEmpty() || !testIdsForSuite.isEmpty()) + { + Root_meta_pure_test_TestSuite testSuite = (Root_meta_pure_test_TestSuite) test; + List updatedTestIds = testIds.isEmpty() + ? testSuite._tests().collect(TestAccessor::_id).toList() + : testIdsForSuite; + debugTestsResult.results.addAll(testRunner.debugTestSuite(testSuite, updatedTestIds, pureModel, data)); + } + } + } + } + return debugTestsResult; + } } diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/extension/TestRunner.java b/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/extension/TestRunner.java index 73c2a828654..6fe0f0781b9 100644 --- a/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/extension/TestRunner.java +++ b/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/extension/TestRunner.java @@ -16,10 +16,12 @@ import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestDebug; import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestResult; import org.finos.legend.pure.generated.Root_meta_pure_test_AtomicTest; import org.finos.legend.pure.generated.Root_meta_pure_test_TestSuite; +import java.util.Collections; import java.util.List; public interface TestRunner @@ -27,4 +29,14 @@ public interface TestRunner TestResult executeAtomicTest(Root_meta_pure_test_AtomicTest atomicTest, PureModel pureModel, PureModelContextData data); List executeTestSuite(Root_meta_pure_test_TestSuite testSuite, List atomicTestIds, PureModel pureModel, PureModelContextData data); + + default TestDebug debugAtomicTest(Root_meta_pure_test_AtomicTest atomicTest, PureModel pureModel, PureModelContextData data) + { + return null; + } + + default List debugTestSuite(Root_meta_pure_test_TestSuite testSuite, List atomicTestIds, PureModel pureModel, PureModelContextData data) + { + return Collections.emptyList(); + } } diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/model/DebugTestsResult.java b/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/model/DebugTestsResult.java new file mode 100644 index 00000000000..8e3453277e2 --- /dev/null +++ b/legend-engine-core/legend-engine-core-testable/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/model/DebugTestsResult.java @@ -0,0 +1,25 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.testable.model; + +import org.finos.legend.engine.protocol.pure.v1.model.test.result.TestDebug; + +import java.util.ArrayList; +import java.util.List; + +public class DebugTestsResult +{ + public List results = new ArrayList<>(); +}