Enable parallel compiler on SLDC mojos (finos#807)

rafaelbey · Jul 9, 2024 · a3ec484 · a3ec484
1 parent 99c3fd1
commit a3ec484
Show file tree

Hide file tree

Showing 6 changed files with 67 additions and 5 deletions.
diff --git a/...-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/file/FileGenerationMojo.java b/...-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/file/FileGenerationMojo.java
@@ -117,7 +117,11 @@ public void execute() throws MojoExecutionException
         }
 
         getLog().info("Compiling model");
-        PureModelBuilder.PureModelWithContextData pureModelWithContextData = pureModelBuilder.withSDLC(buildSDLCInfo()).withProtocol(buildProtocol()).build();
+        PureModelBuilder.PureModelWithContextData pureModelWithContextData = pureModelBuilder
+                .withSDLC(buildSDLCInfo())
+                .withProtocol(buildProtocol())
+                .withParallelCompiler(true)
+                .build();
         PureModelContextData pureModelContextData = pureModelWithContextData.getPureModelContextData();
         PureModel pureModel = pureModelWithContextData.getPureModel();
         long modelEnd = System.nanoTime();

diff --git a/...aven-plugin/src/main/java/org/finos/legend/sdlc/generation/model/ModelGenerationMojo.java b/...aven-plugin/src/main/java/org/finos/legend/sdlc/generation/model/ModelGenerationMojo.java
@@ -83,7 +83,8 @@ public void execute() throws MojoExecutionException
         // Load Model
         long modelStart = System.nanoTime();
         getLog().info("Start loading model");
-        PureModelBuilder pureModelBuilder = PureModelBuilder.newBuilder();
+        PureModelBuilder pureModelBuilder = PureModelBuilder.newBuilder()
+                .withParallelCompiler(true);
         try (EntityLoader allEntities = EntityLoader.newEntityLoader(Thread.currentThread().getContextClassLoader()))
         {
             pureModelBuilder.addEntitiesIfPossible(allEntities.getAllEntities());

diff --git a/...plugin/src/main/java/org/finos/legend/sdlc/generation/service/ServicesGenerationMojo.java b/...plugin/src/main/java/org/finos/legend/sdlc/generation/service/ServicesGenerationMojo.java
@@ -113,7 +113,8 @@ public void execute() throws MojoExecutionException
         getLog().info("Loading model");
         long modelStart = System.nanoTime();
 
-        PureModelBuilder pureModelBuilder = PureModelBuilder.newBuilder();
+        PureModelBuilder pureModelBuilder = PureModelBuilder.newBuilder()
+                .withParallelCompiler(true);
         try (EntityLoader allEntities = EntityLoader.newEntityLoader(Thread.currentThread().getContextClassLoader()))
         {
             pureModelBuilder.addEntitiesIfPossible(allEntities.getAllEntities());

diff --git a/.../main/java/org/finos/legend/sdlc/language/pure/compiler/toPureGraph/PureModelBuilder.java b/.../main/java/org/finos/legend/sdlc/language/pure/compiler/toPureGraph/PureModelBuilder.java
@@ -27,6 +27,8 @@
 import org.finos.legend.sdlc.protocol.pure.v1.EntityToPureConverter;
 import org.finos.legend.sdlc.protocol.pure.v1.PureModelContextDataBuilder;
 
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinWorkerThread;
 import java.util.ServiceLoader;
 import java.util.stream.Stream;
 
@@ -36,6 +38,7 @@ public class PureModelBuilder
     private ClassLoader classLoader;
     private CompilerExtensions extensions;
     private String packagePrefix;
+    private boolean parallelCompiler;
 
     private PureModelBuilder(EntityToPureConverter converter)
     {
@@ -217,7 +220,49 @@ public PureModel buildPureModel(ClassLoader classLoader)
 
     private PureModel buildPureModel(PureModelContextData pureModelContextData)
     {
-        return new PureModel(pureModelContextData, getExtensions(), null, this.classLoader, DeploymentMode.PROD, new PureModelProcessParameter(this.packagePrefix), null);
+        ForkJoinPool pool = null;
+
+        if (this.parallelCompiler)
+        {
+            int parallelism = Runtime.getRuntime().availableProcessors();
+
+            if (this.classLoader != null)
+            {
+                pool = new ForkJoinPool(
+                        parallelism,
+                        p ->
+                        {
+                            ForkJoinWorkerThread workerThread = new ForkJoinWorkerThread(p)
+                            {
+
+                            };
+                            workerThread.setContextClassLoader(this.classLoader);
+                            return workerThread;
+                        },
+                        null,
+                        false
+                );
+            }
+            else
+            {
+                pool = new ForkJoinPool(parallelism);
+            }
+        }
+        PureModelProcessParameter pureModelProcessParameter = PureModelProcessParameter.newBuilder()
+                .withPackagePrefix(this.packagePrefix)
+                .withForkJoinPool(pool)
+                .build();
+        try
+        {
+            return new PureModel(pureModelContextData, getExtensions(), null, this.classLoader, DeploymentMode.PROD, pureModelProcessParameter, null);
+        }
+        finally
+        {
+            if (pool != null)
+            {
+                pool.shutdown();
+            }
+        }
     }
 
     private CompilerExtensions getExtensions()
@@ -240,6 +285,12 @@ public static PureModelBuilder newBuilder(EntityToPureConverter converter)
         return new PureModelBuilder(converter);
     }
 
+    public PureModelBuilder withParallelCompiler(boolean parallelCompiler)
+    {
+        this.parallelCompiler = parallelCompiler;
+        return this;
+    }
+
     public static class PureModelWithContextData
     {
         private final PureModel pureModel;

diff --git a/...test-utils/src/main/java/org/finos/legend/sdlc/test/junit/LegendSDLCTestSuiteBuilder.java b/...test-utils/src/main/java/org/finos/legend/sdlc/test/junit/LegendSDLCTestSuiteBuilder.java
@@ -73,7 +73,11 @@ private LegendSDLCTestSuiteBuilder(String name, String pureVersion, ClassLoader
         this.pureVersion = pureVersion;
         this.testableClassifiers = TestableRunnerExtensionLoader.getClassifierPathToTestableRunnerMap(classLoader).keySet();
         this.entities = getEntities(classLoader);
-        PureModelWithContextData pureModelWithContextData = PureModelBuilder.newBuilder().withEntitiesIfPossible(this.entities).withClassLoader(classLoader).build();
+        PureModelWithContextData pureModelWithContextData = PureModelBuilder.newBuilder()
+                .withEntitiesIfPossible(this.entities)
+                .withClassLoader(classLoader)
+                .withParallelCompiler(true)
+                .build();
         this.pureModel = pureModelWithContextData.getPureModel();
         this.pureModelContextData = pureModelWithContextData.getPureModelContextData();
         this.protocolIndex = Iterate.groupByUniqueKey(this.pureModelContextData.getElements(), PackageableElement::getPath);

diff --git a/...dlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/PureTestHelper.java b/...dlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/PureTestHelper.java
@@ -121,6 +121,7 @@ private static PureModelWithContextData buildPureModelWithContextData(ClassLoade
             return PureModelBuilder.newBuilder()
                     .withEntitiesIfPossible(entityLoader.getAllEntities())
                     .withClassLoader(classLoader)
+                    .withParallelCompiler(true)
                     .build();
         }
         catch (Exception e)