Add parallel compiler to REPL (#2976)

* Add parallel compiler to REPL * Add parallel compiler to REPL
finos · Jul 24, 2024 · df47a22 · df47a22
1 parent cc1e0ea
commit df47a22
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 9 deletions.
diff --git a/...engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java b/...engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java
@@ -46,6 +46,7 @@
 import org.finos.legend.engine.repl.autocomplete.handlers.SelectHandler;
 import org.finos.legend.engine.repl.autocomplete.handlers.SortHandler;
 import org.finos.legend.engine.repl.autocomplete.parser.ParserFixer;
+import org.finos.legend.engine.repl.core.legend.LegendInterface;
 import org.finos.legend.engine.shared.core.identity.Identity;
 import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionAccessor;
@@ -69,7 +70,7 @@ public class Completer
     private final String header;
     private final int lineOffset;
     private final MutableMap<String, FunctionHandler> handlers;
-
+    private final java.util.function.Function<PureModelContextData, PureModel> compiler;
     private MutableList<CompleterExtension> extensions;
 
     public Completer(String buildCodeContext)
@@ -79,6 +80,17 @@ public Completer(String buildCodeContext)
 
     public Completer(String buildCodeContext, MutableList<CompleterExtension> extensions)
     {
+        this(buildCodeContext, extensions, x -> Compiler.compile(x, null, Identity.getAnonymousIdentity().getName()));
+    }
+
+    public Completer(String buildCodeContext, MutableList<CompleterExtension> extensions, LegendInterface legendInterface)
+    {
+        this(buildCodeContext, extensions, legendInterface::compile);
+    }
+
+    private Completer(String buildCodeContext, MutableList<CompleterExtension> extensions, java.util.function.Function<PureModelContextData, PureModel> compiler)
+    {
+        this.compiler = compiler;
         this.extensions = extensions;
         this.buildCodeContext = buildCodeContext;
         this.header =
@@ -125,7 +137,7 @@ public CompletionResult complete(String value)
             ValueSpecification currentExpression = findPartiallyWrittenExpression(vs, lineOffset, value.length());
 
             PureModelContextData pureModelContextData = PureGrammarParser.newInstance().parseModel(buildCodeContext);
-            PureModel pureModel = Compiler.compile(pureModelContextData, null, Identity.getAnonymousIdentity().getName());
+            PureModel pureModel = this.compiler.apply(pureModelContextData);
 
             ProcessingContext processingContext = new ProcessingContext("");
 

diff --git a/...l-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java b/...l-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java
@@ -14,10 +14,12 @@
 
 package org.finos.legend.engine.repl.core.legend;
 
+import java.util.concurrent.ForkJoinPool;
 import org.eclipse.collections.api.RichIterable;
 import org.eclipse.collections.api.tuple.Pair;
 import org.finos.legend.engine.language.pure.compiler.Compiler;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
+import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModelProcessParameter;
 import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser;
 import org.finos.legend.engine.plan.generation.PlanGenerator;
 import org.finos.legend.engine.plan.platform.PlanPlatform;
@@ -34,6 +36,8 @@
 
 public class LocalLegendInterface implements LegendInterface
 {
+    private final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors());
+
     @Override
     public PureModelContextData parse(String txt)
     {
@@ -77,7 +81,7 @@ public PureModelContextData parse(String txt)
     @Override
     public PureModel compile(PureModelContextData pureModelContextData)
     {
-        return Compiler.compile(pureModelContextData, DeploymentMode.PROD, Identity.getAnonymousIdentity().getName());
+        return Compiler.compile(pureModelContextData, DeploymentMode.PROD, Identity.getAnonymousIdentity().getName(), null, PureModelProcessParameter.newBuilder().withForkJoinPool(this.forkJoinPool).build());
     }
 
     @Override

diff --git a/...ational/src/main/java/org/finos/legend/engine/repl/relational/server/DataCubeHelpers.java b/...ational/src/main/java/org/finos/legend/engine/repl/relational/server/DataCubeHelpers.java
@@ -110,7 +110,7 @@ public static String getQueryCode(ValueSpecification valueSpecification, Boolean
         return valueSpecification.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().withRenderStyle(pretty != null && pretty ? RenderStyle.PRETTY : RenderStyle.STANDARD).build());
     }
 
-    public static CompletionResult getCodeTypeahead(String code, Boolean isPartial, PureModelContextData data)
+    public static CompletionResult getCodeTypeahead(LegendInterface legendInterface, String code, Boolean isPartial, PureModelContextData data)
     {
         try
         {
@@ -127,7 +127,7 @@ public static CompletionResult getCodeTypeahead(String code, Boolean isPartial,
                 String existingCode = func.body.get(0).accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build());
                 queryCode = existingCode + code;
             }
-            CompletionResult result = new Completer(graphCode, Lists.mutable.with(new RelationalCompleterExtension())).complete(queryCode);
+            CompletionResult result = new Completer(graphCode, Lists.mutable.with(new RelationalCompleterExtension()), legendInterface).complete(queryCode);
             if (result.getEngineException() != null)
             {
                 return new CompletionResult(Lists.mutable.empty());

diff --git a/...ain/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryBuilder.java b/...ain/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryBuilder.java
@@ -140,7 +140,7 @@ public HttpHandler getHandler(REPLServerState state)
                         String requestBody = bufferReader.lines().collect(Collectors.joining());
                         DataCubeQueryTypeaheadInput input = state.objectMapper.readValue(requestBody, DataCubeQueryTypeaheadInput.class);
                         PureModelContextData data = state.getCurrentPureModelContextData();
-                        CompletionResult result = DataCubeHelpers.getCodeTypeahead(input.code, input.isPartial, data);
+                        CompletionResult result = DataCubeHelpers.getCodeTypeahead(state.legendInterface, input.code, input.isPartial, data);
                         handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result.getCompletion()), state);
                     }
                     catch (Exception e)

diff --git a/...ngine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestDataCubeHelpers.java b/...ngine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestDataCubeHelpers.java
@@ -214,7 +214,7 @@ private void testTypeahead(String expectedResult, String code, boolean isPartial
     {
         try
         {
-            Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getCodeTypeahead(code, isPartial, pureModelContextData)));
+            Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getCodeTypeahead(this.legendInterface, code, isPartial, pureModelContextData)));
         }
         catch (IOException e)
         {

diff --git a/...legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java b/...legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java
@@ -270,8 +270,7 @@ public void run(T serverConfiguration, Environment environment)
         DeploymentStateAndVersions.DEPLOYMENT_MODE = serverConfiguration.deployment.mode;
 
         SDLCLoader sdlcLoader = new SDLCLoader(serverConfiguration.metadataserver, null);
-        ForkJoinPool compilerPool = new ForkJoinPool();
-        ModelManager modelManager = new ModelManager(serverConfiguration.deployment.mode, compilerPool, sdlcLoader);
+        ModelManager modelManager = new ModelManager(serverConfiguration.deployment.mode, sdlcLoader);
 
         ChainFixingFilterHandler.apply(environment.getApplicationContext(), serverConfiguration.filterPriorities);
-Original file line number
+Diff line change
@@ Expand Up @@
         {
             try
             {
-                Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getCodeTypeahead(code, isPartial, pureModelContextData)));
+                Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getCodeTypeahead(this.legendInterface, code, isPartial, pureModelContextData)));
             }
             catch (IOException e)
             {
@@ Expand Down @@