From 419d32202fc94f23fc4e69972b10a12306fc42dc Mon Sep 17 00:00:00 2001 From: Rafael Bey Date: Sat, 6 Jul 2024 00:53:14 -0400 Subject: [PATCH] Enable parallel compiler on SLDC mojos --- .../generation/file/FileGenerationMojo.java | 6 ++- .../generation/model/ModelGenerationMojo.java | 3 +- .../service/ServicesGenerationMojo.java | 3 +- .../toPureGraph/PureModelBuilder.java | 53 ++++++++++++++++++- .../junit/LegendSDLCTestSuiteBuilder.java | 6 ++- .../test/junit/pure/v1/PureTestHelper.java | 1 + 6 files changed, 67 insertions(+), 5 deletions(-) diff --git a/legend-sdlc-generation-file-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/file/FileGenerationMojo.java b/legend-sdlc-generation-file-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/file/FileGenerationMojo.java index ceb80c1b5e..28ed8b9af8 100644 --- a/legend-sdlc-generation-file-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/file/FileGenerationMojo.java +++ b/legend-sdlc-generation-file-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/file/FileGenerationMojo.java @@ -117,7 +117,11 @@ public void execute() throws MojoExecutionException } getLog().info("Compiling model"); - PureModelBuilder.PureModelWithContextData pureModelWithContextData = pureModelBuilder.withSDLC(buildSDLCInfo()).withProtocol(buildProtocol()).build(); + PureModelBuilder.PureModelWithContextData pureModelWithContextData = pureModelBuilder + .withSDLC(buildSDLCInfo()) + .withProtocol(buildProtocol()) + .withParallelCompiler(true) + .build(); PureModelContextData pureModelContextData = pureModelWithContextData.getPureModelContextData(); PureModel pureModel = pureModelWithContextData.getPureModel(); long modelEnd = System.nanoTime(); diff --git a/legend-sdlc-generation-model-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/model/ModelGenerationMojo.java b/legend-sdlc-generation-model-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/model/ModelGenerationMojo.java index 2bbc408f2b..8590099762 100644 --- a/legend-sdlc-generation-model-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/model/ModelGenerationMojo.java +++ b/legend-sdlc-generation-model-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/model/ModelGenerationMojo.java @@ -83,7 +83,8 @@ public void execute() throws MojoExecutionException // Load Model long modelStart = System.nanoTime(); getLog().info("Start loading model"); - PureModelBuilder pureModelBuilder = PureModelBuilder.newBuilder(); + PureModelBuilder pureModelBuilder = PureModelBuilder.newBuilder() + .withParallelCompiler(true); try (EntityLoader allEntities = EntityLoader.newEntityLoader(Thread.currentThread().getContextClassLoader())) { pureModelBuilder.addEntitiesIfPossible(allEntities.getAllEntities()); diff --git a/legend-sdlc-generation-service-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/service/ServicesGenerationMojo.java b/legend-sdlc-generation-service-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/service/ServicesGenerationMojo.java index 8de45ed1c1..c7fc4039a7 100644 --- a/legend-sdlc-generation-service-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/service/ServicesGenerationMojo.java +++ b/legend-sdlc-generation-service-maven-plugin/src/main/java/org/finos/legend/sdlc/generation/service/ServicesGenerationMojo.java @@ -113,7 +113,8 @@ public void execute() throws MojoExecutionException getLog().info("Loading model"); long modelStart = System.nanoTime(); - PureModelBuilder pureModelBuilder = PureModelBuilder.newBuilder(); + PureModelBuilder pureModelBuilder = PureModelBuilder.newBuilder() + .withParallelCompiler(true); try (EntityLoader allEntities = EntityLoader.newEntityLoader(Thread.currentThread().getContextClassLoader())) { pureModelBuilder.addEntitiesIfPossible(allEntities.getAllEntities()); diff --git a/legend-sdlc-language-pure-compiler/src/main/java/org/finos/legend/sdlc/language/pure/compiler/toPureGraph/PureModelBuilder.java b/legend-sdlc-language-pure-compiler/src/main/java/org/finos/legend/sdlc/language/pure/compiler/toPureGraph/PureModelBuilder.java index e7a59c745e..c5e56e13f5 100644 --- a/legend-sdlc-language-pure-compiler/src/main/java/org/finos/legend/sdlc/language/pure/compiler/toPureGraph/PureModelBuilder.java +++ b/legend-sdlc-language-pure-compiler/src/main/java/org/finos/legend/sdlc/language/pure/compiler/toPureGraph/PureModelBuilder.java @@ -27,6 +27,8 @@ import org.finos.legend.sdlc.protocol.pure.v1.EntityToPureConverter; import org.finos.legend.sdlc.protocol.pure.v1.PureModelContextDataBuilder; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinWorkerThread; import java.util.ServiceLoader; import java.util.stream.Stream; @@ -36,6 +38,7 @@ public class PureModelBuilder private ClassLoader classLoader; private CompilerExtensions extensions; private String packagePrefix; + private boolean parallelCompiler; private PureModelBuilder(EntityToPureConverter converter) { @@ -217,7 +220,49 @@ public PureModel buildPureModel(ClassLoader classLoader) private PureModel buildPureModel(PureModelContextData pureModelContextData) { - return new PureModel(pureModelContextData, getExtensions(), null, this.classLoader, DeploymentMode.PROD, new PureModelProcessParameter(this.packagePrefix), null); + ForkJoinPool pool = null; + + if (this.parallelCompiler) + { + int parallelism = Runtime.getRuntime().availableProcessors(); + + if (this.classLoader != null) + { + pool = new ForkJoinPool( + parallelism, + p -> + { + ForkJoinWorkerThread workerThread = new ForkJoinWorkerThread(p) + { + + }; + workerThread.setContextClassLoader(this.classLoader); + return workerThread; + }, + null, + false + ); + } + else + { + pool = new ForkJoinPool(parallelism); + } + } + PureModelProcessParameter pureModelProcessParameter = PureModelProcessParameter.newBuilder() + .withPackagePrefix(this.packagePrefix) + .withForkJoinPool(pool) + .build(); + try + { + return new PureModel(pureModelContextData, getExtensions(), null, this.classLoader, DeploymentMode.PROD, pureModelProcessParameter, null); + } + finally + { + if (pool != null) + { + pool.shutdown(); + } + } } private CompilerExtensions getExtensions() @@ -240,6 +285,12 @@ public static PureModelBuilder newBuilder(EntityToPureConverter converter) return new PureModelBuilder(converter); } + public PureModelBuilder withParallelCompiler(boolean parallelCompiler) + { + this.parallelCompiler = parallelCompiler; + return this; + } + public static class PureModelWithContextData { private final PureModel pureModel; diff --git a/legend-sdlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/LegendSDLCTestSuiteBuilder.java b/legend-sdlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/LegendSDLCTestSuiteBuilder.java index 648fd66c21..c6cd8bb597 100644 --- a/legend-sdlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/LegendSDLCTestSuiteBuilder.java +++ b/legend-sdlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/LegendSDLCTestSuiteBuilder.java @@ -73,7 +73,11 @@ private LegendSDLCTestSuiteBuilder(String name, String pureVersion, ClassLoader this.pureVersion = pureVersion; this.testableClassifiers = TestableRunnerExtensionLoader.getClassifierPathToTestableRunnerMap(classLoader).keySet(); this.entities = getEntities(classLoader); - PureModelWithContextData pureModelWithContextData = PureModelBuilder.newBuilder().withEntitiesIfPossible(this.entities).withClassLoader(classLoader).build(); + PureModelWithContextData pureModelWithContextData = PureModelBuilder.newBuilder() + .withEntitiesIfPossible(this.entities) + .withClassLoader(classLoader) + .withParallelCompiler(true) + .build(); this.pureModel = pureModelWithContextData.getPureModel(); this.pureModelContextData = pureModelWithContextData.getPureModelContextData(); this.protocolIndex = Iterate.groupByUniqueKey(this.pureModelContextData.getElements(), PackageableElement::getPath); diff --git a/legend-sdlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/PureTestHelper.java b/legend-sdlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/PureTestHelper.java index 199d6001e7..d6ec82cf4a 100644 --- a/legend-sdlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/PureTestHelper.java +++ b/legend-sdlc-test-utils/src/main/java/org/finos/legend/sdlc/test/junit/pure/v1/PureTestHelper.java @@ -121,6 +121,7 @@ private static PureModelWithContextData buildPureModelWithContextData(ClassLoade return PureModelBuilder.newBuilder() .withEntitiesIfPossible(entityLoader.getAllEntities()) .withClassLoader(classLoader) + .withParallelCompiler(true) .build(); } catch (Exception e)