Change parallelization of service execution generation (#699)

finos · Aug 28, 2023 · 8a4da68 · 8a4da68
1 parent 51ca74d
commit 8a4da68
Show file tree

Hide file tree

Showing 7 changed files with 199 additions and 107 deletions.
diff --git a/...plugin/src/main/java/org/finos/legend/sdlc/generation/service/ServicesGenerationMojo.java b/...plugin/src/main/java/org/finos/legend/sdlc/generation/service/ServicesGenerationMojo.java
@@ -35,8 +35,9 @@
 import org.finos.legend.engine.plan.generation.extension.PlanGeneratorExtension;
 import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
+import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.service.PureMultiExecution;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.service.Service;
-import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader;
+import org.finos.legend.engine.pure.code.core.PureCoreExtension;
 import org.finos.legend.sdlc.domain.model.entity.Entity;
 import org.finos.legend.sdlc.language.pure.compiler.toPureGraph.PureModelBuilder;
 import org.finos.legend.sdlc.protocol.pure.v1.EntityToPureConverter;
@@ -235,12 +236,13 @@ private void generateServices(MutableMap<String, Service> servicesByPath, PureMo
                 .serializationInclusion(JsonInclude.Include.NON_NULL)
                 .build());
 
-        int effectiveParallelism = Math.min(parallelism, servicesByPath.size());
         ForkJoinPool pool;
-        if (effectiveParallelism > 1)
+        // We only create a pool if parallelism level is greater than 1 and we expect to generate multiple plans, which
+        // is true if we have multiple services or we have one or more a multi-execution service.
+        if ((parallelism > 1) && ((servicesByPath.size() > 1) || servicesByPath.anySatisfy(s -> s.execution instanceof PureMultiExecution)))
         {
-            getLog().info("Generating services in parallel with parallelism level " + effectiveParallelism);
-            pool = createForkJoinPool(effectiveParallelism);
+            getLog().info("Generating services in parallel with parallelism level " + parallelism);
+            pool = createForkJoinPool(parallelism);
         }
         else
         {
@@ -255,7 +257,7 @@ private void generateServices(MutableMap<String, Service> servicesByPath, PureMo
                     .withOutputDirectories(this.javaSourceOutputDirectory.toPath(), this.resourceOutputDirectory.toPath())
                     .withJsonMapper(jsonMapper)
                     .withPlanGeneratorExtensions(ServiceLoader.load(PlanGeneratorExtension.class))
-                    .withPureCoreExtensions(PureCoreExtensionLoader.extensions())
+                    .withPureCoreExtensions(ServiceLoader.load(PureCoreExtension.class))
                     .withExecutorService(pool)
                     .build()
                     .generate();

diff --git a/...ice/src/main/java/org/finos/legend/sdlc/generation/service/ServiceExecutionGenerator.java b/...ice/src/main/java/org/finos/legend/sdlc/generation/service/ServiceExecutionGenerator.java
diff --git a/...src/test/java/org/finos/legend/sdlc/generation/service/TestServiceExecutionGenerator.java b/...src/test/java/org/finos/legend/sdlc/generation/service/TestServiceExecutionGenerator.java
@@ -19,7 +19,6 @@
 import io.github.classgraph.ClassGraph;
 import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.factory.Sets;
-import org.eclipse.collections.api.list.ImmutableList;
 import org.eclipse.collections.impl.utility.Iterate;
 import org.eclipse.collections.impl.utility.LazyIterate;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
@@ -38,19 +37,22 @@
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.service.PureExecution;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.service.Service;
-import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader;
+import org.finos.legend.engine.pure.code.core.PureCoreExtension;
 import org.finos.legend.engine.shared.core.url.StreamProvider;
-import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;
 import org.finos.legend.pure.m3.navigation.PrimitiveUtilities;
 import org.finos.legend.pure.runtime.java.compiled.compiler.MemoryFileManager;
 import org.finos.legend.pure.runtime.java.compiled.generation.orchestrator.VoidLog;
 import org.finos.legend.sdlc.language.pure.compiler.toPureGraph.PureModelBuilder;
 import org.finos.legend.sdlc.serialization.EntityLoader;
 import org.finos.legend.sdlc.tools.entity.EntityPaths;
-import org.junit.*;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
-import javax.tools.*;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -68,9 +70,23 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.temporal.Temporal;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.ServiceLoader;
+import java.util.Set;
+import java.util.Spliterator;
+import java.util.Spliterators;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
+import javax.tools.DiagnosticCollector;
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+import javax.tools.ToolProvider;
 
 public class TestServiceExecutionGenerator
 {
@@ -403,12 +419,11 @@ private ClassLoader generateAndCompile(String packagePrefix, Collection<? extend
                 .withPackagePrefix(packagePrefix)
                 .withOutputDirectories(this.generatedSourcesDirectory, this.classesDirectory)
                 .withPlanGeneratorExtensions(ServiceLoader.load(PlanGeneratorExtension.class))
-                .withPureCoreExtensions(PureCoreExtensionLoader.extensions())
+                .withPureCoreExtensions(ServiceLoader.load(PureCoreExtension.class))
                 .withClientVersion("vX_X_X")
                 .build();
         generator.generate();
 
-        ImmutableList<? extends Root_meta_pure_extension_Extension> extensions = Lists.mutable.withAll(PureCoreExtensionLoader.extensions()).flatCollect(e -> e.extraPureCoreExtensions(PURE_MODEL.getExecutionSupport())).toImmutable();
         // Generate
         Set<String> enumClasses = LazyIterate.flatCollect(services, s -> ((PureExecution) s.execution).func.parameters)
                 .collectIf(

diff --git a/...test-utils/src/main/java/org/finos/legend/sdlc/test/junit/LegendSDLCTestSuiteBuilder.java b/...test-utils/src/main/java/org/finos/legend/sdlc/test/junit/LegendSDLCTestSuiteBuilder.java
@@ -32,7 +32,7 @@
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.service.Service;
-import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader;
+import org.finos.legend.engine.pure.code.core.PureCoreExtension;
 import org.finos.legend.engine.testable.extension.TestableRunnerExtensionLoader;
 import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.test.Test;
@@ -65,7 +65,7 @@ public class LegendSDLCTestSuiteBuilder
     private final PureModelContextData pureModelContextData;
     private final MapIterable<String, PackageableElement> protocolIndex;
     private final RichIterable<? extends Root_meta_pure_extension_Extension> routerExtensions;
-    private final MutableList<PlanTransformer> planTransformers;
+    private final Iterable<? extends PlanTransformer> planTransformers;
 
     private LegendSDLCTestSuiteBuilder(String name, String pureVersion, ClassLoader classLoader)
     {
@@ -77,9 +77,8 @@ private LegendSDLCTestSuiteBuilder(String name, String pureVersion, ClassLoader
         this.pureModel = pureModelWithContextData.getPureModel();
         this.pureModelContextData = pureModelWithContextData.getPureModelContextData();
         this.protocolIndex = Iterate.groupByUniqueKey(this.pureModelContextData.getElements(), PackageableElement::getPath);
-        MutableList<PlanGeneratorExtension> extensions = Lists.mutable.withAll(ServiceLoader.load(PlanGeneratorExtension.class, classLoader));
-        this.routerExtensions = PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(this.pureModel.getExecutionSupport()));
-        this.planTransformers = extensions.flatCollect(PlanGeneratorExtension::getExtraPlanTransformers);
+        this.routerExtensions = Iterate.flatCollect(ServiceLoader.load(PureCoreExtension.class, classLoader), e -> e.extraPureCoreExtensions(this.pureModel.getExecutionSupport()), Lists.mutable.empty());
+        this.planTransformers = Iterate.flatCollect(ServiceLoader.load(PlanGeneratorExtension.class, classLoader), PlanGeneratorExtension::getExtraPlanTransformers, Lists.mutable.empty());
     }
 
     public LegendSDLCTestSuiteBuilder(String name, String pureVersion)

diff --git a/...t-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/LegacyMappingTestCase.java b/...t-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/LegacyMappingTestCase.java
@@ -15,7 +15,6 @@
 package org.finos.legend.sdlc.test.junit.pure.v1;
 
 import org.eclipse.collections.api.RichIterable;
-import org.eclipse.collections.api.list.MutableList;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
 import org.finos.legend.engine.plan.execution.PlanExecutor;
 import org.finos.legend.engine.plan.generation.transformers.PlanTransformer;
@@ -28,7 +27,7 @@ public class LegacyMappingTestCase extends LegendSDLCTestCase
 {
     private final LegacyMappingTestHelper helper;
 
-    public LegacyMappingTestCase(String mappingPath, PureModel pureModel, MappingTest_Legacy mappingTest, MutableList<PlanTransformer> planTransformers, RichIterable<? extends Root_meta_pure_extension_Extension> extensions, String pureVersion)
+    public LegacyMappingTestCase(String mappingPath, PureModel pureModel, MappingTest_Legacy mappingTest, Iterable<? extends PlanTransformer> planTransformers, RichIterable<? extends Root_meta_pure_extension_Extension> extensions, String pureVersion)
     {
         super(mappingPath);
         this.helper = new LegacyMappingTestHelper(3, mappingPath, new MappingTestRunner(pureModel, mappingPath, mappingTest, PlanExecutor.newPlanExecutorBuilder().withAvailableStoreExecutors().build(), extensions, planTransformers, pureVersion));

diff --git a/...t-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/LegacyServiceTestCase.java b/...t-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/LegacyServiceTestCase.java
@@ -15,7 +15,6 @@
 package org.finos.legend.sdlc.test.junit.pure.v1;
 
 import org.eclipse.collections.api.RichIterable;
-import org.eclipse.collections.api.list.MutableList;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
 import org.finos.legend.engine.plan.execution.PlanExecutor;
 import org.finos.legend.engine.plan.generation.transformers.PlanTransformer;
@@ -29,7 +28,7 @@ public class LegacyServiceTestCase extends LegendSDLCTestCase
 {
     private final LegacyServiceTestHelper helper;
 
-    public LegacyServiceTestCase(String servicePath, PureModel pureModel, PureModelContextData pureModelContextData, Service service, MutableList<PlanTransformer> planTransformers, RichIterable<? extends Root_meta_pure_extension_Extension> extensions, String pureVersion)
+    public LegacyServiceTestCase(String servicePath, PureModel pureModel, PureModelContextData pureModelContextData, Service service, Iterable<? extends PlanTransformer> planTransformers, RichIterable<? extends Root_meta_pure_extension_Extension> extensions, String pureVersion)
     {
         super(servicePath);
         this.helper = new LegacyServiceTestHelper(3, servicePath, new ServiceTestRunner(service, null, pureModelContextData, pureModel, null, PlanExecutor.newPlanExecutorBuilder().withAvailableStoreExecutors().build(), extensions, planTransformers, pureVersion));

diff --git a/...dlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/PureTestHelper.java b/...dlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/PureTestHelper.java
@@ -20,12 +20,13 @@
 import org.eclipse.collections.api.list.MutableList;
 import org.eclipse.collections.api.map.MapIterable;
 import org.eclipse.collections.api.map.MutableMap;
+import org.eclipse.collections.impl.utility.Iterate;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
 import org.finos.legend.engine.plan.generation.extension.PlanGeneratorExtension;
 import org.finos.legend.engine.plan.generation.transformers.PlanTransformer;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement;
-import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader;
+import org.finos.legend.engine.pure.code.core.PureCoreExtension;
 import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;
 import org.finos.legend.sdlc.language.pure.compiler.toPureGraph.PureModelBuilder;
 import org.finos.legend.sdlc.language.pure.compiler.toPureGraph.PureModelBuilder.PureModelWithContextData;
@@ -94,9 +95,8 @@ static void initialize()
                     PROTOCOL_ELEMENT_INDEX = indexPureModelContextData(PURE_MODEL_CONTEXT_DATA);
                     PURE_MODEL = pureModelWithContextData.getPureModel();
 
-                    MutableList<PlanGeneratorExtension> extensions = Lists.mutable.withAll(ServiceLoader.load(PlanGeneratorExtension.class));
-                    PLAN_TRANSFORMERS = extensions.flatCollect(PlanGeneratorExtension::getExtraPlanTransformers).asUnmodifiable();
-                    ROUTER_EXTENSIONS = PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(PURE_MODEL.getExecutionSupport())).asUnmodifiable();
+                    PLAN_TRANSFORMERS = Iterate.flatCollect(ServiceLoader.load(PlanGeneratorExtension.class, classLoader), PlanGeneratorExtension::getExtraPlanTransformers, Lists.mutable.empty()).asUnmodifiable();
+                    ROUTER_EXTENSIONS = Iterate.flatCollect(ServiceLoader.load(PureCoreExtension.class, classLoader), e -> e.extraPureCoreExtensions(PURE_MODEL.getExecutionSupport()), Lists.mutable.empty()).asUnmodifiable();
                     LOGGER.debug("Finished initialization");
                 }
                 catch (Throwable t)