From df47a22fc4233fe4d4526acdb4278a12424b2fa9 Mon Sep 17 00:00:00 2001 From: Rafael Bey <24432403+rafaelbey@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:33:07 -0400 Subject: [PATCH] Add parallel compiler to REPL (#2976) * Add parallel compiler to REPL * Add parallel compiler to REPL --- .../engine/repl/autocomplete/Completer.java | 16 ++++++++++++++-- .../repl/core/legend/LocalLegendInterface.java | 6 +++++- .../repl/relational/server/DataCubeHelpers.java | 4 ++-- .../server/handler/DataCubeQueryBuilder.java | 2 +- .../legend/engine/repl/TestDataCubeHelpers.java | 2 +- .../org/finos/legend/engine/server/Server.java | 3 +-- 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java index 341e26b2685..13e9162a0e9 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java @@ -46,6 +46,7 @@ import org.finos.legend.engine.repl.autocomplete.handlers.SelectHandler; import org.finos.legend.engine.repl.autocomplete.handlers.SortHandler; import org.finos.legend.engine.repl.autocomplete.parser.ParserFixer; +import org.finos.legend.engine.repl.core.legend.LegendInterface; import org.finos.legend.engine.shared.core.identity.Identity; import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionAccessor; @@ -69,7 +70,7 @@ public class Completer private final String header; private final int lineOffset; private final MutableMap handlers; - + private final java.util.function.Function compiler; private MutableList extensions; public Completer(String buildCodeContext) @@ -79,6 +80,17 @@ public Completer(String buildCodeContext) public Completer(String buildCodeContext, MutableList extensions) { + this(buildCodeContext, extensions, x -> Compiler.compile(x, null, Identity.getAnonymousIdentity().getName())); + } + + public Completer(String buildCodeContext, MutableList extensions, LegendInterface legendInterface) + { + this(buildCodeContext, extensions, legendInterface::compile); + } + + private Completer(String buildCodeContext, MutableList extensions, java.util.function.Function compiler) + { + this.compiler = compiler; this.extensions = extensions; this.buildCodeContext = buildCodeContext; this.header = @@ -125,7 +137,7 @@ public CompletionResult complete(String value) ValueSpecification currentExpression = findPartiallyWrittenExpression(vs, lineOffset, value.length()); PureModelContextData pureModelContextData = PureGrammarParser.newInstance().parseModel(buildCodeContext); - PureModel pureModel = Compiler.compile(pureModelContextData, null, Identity.getAnonymousIdentity().getName()); + PureModel pureModel = this.compiler.apply(pureModelContextData); ProcessingContext processingContext = new ProcessingContext(""); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java index a14a2c8afbd..68574ae3f24 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java @@ -14,10 +14,12 @@ package org.finos.legend.engine.repl.core.legend; +import java.util.concurrent.ForkJoinPool; import org.eclipse.collections.api.RichIterable; import org.eclipse.collections.api.tuple.Pair; import org.finos.legend.engine.language.pure.compiler.Compiler; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModelProcessParameter; import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser; import org.finos.legend.engine.plan.generation.PlanGenerator; import org.finos.legend.engine.plan.platform.PlanPlatform; @@ -34,6 +36,8 @@ public class LocalLegendInterface implements LegendInterface { + private final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors()); + @Override public PureModelContextData parse(String txt) { @@ -77,7 +81,7 @@ public PureModelContextData parse(String txt) @Override public PureModel compile(PureModelContextData pureModelContextData) { - return Compiler.compile(pureModelContextData, DeploymentMode.PROD, Identity.getAnonymousIdentity().getName()); + return Compiler.compile(pureModelContextData, DeploymentMode.PROD, Identity.getAnonymousIdentity().getName(), null, PureModelProcessParameter.newBuilder().withForkJoinPool(this.forkJoinPool).build()); } @Override diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/DataCubeHelpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/DataCubeHelpers.java index 090a050a8f9..e1fa89a0557 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/DataCubeHelpers.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/DataCubeHelpers.java @@ -110,7 +110,7 @@ public static String getQueryCode(ValueSpecification valueSpecification, Boolean return valueSpecification.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().withRenderStyle(pretty != null && pretty ? RenderStyle.PRETTY : RenderStyle.STANDARD).build()); } - public static CompletionResult getCodeTypeahead(String code, Boolean isPartial, PureModelContextData data) + public static CompletionResult getCodeTypeahead(LegendInterface legendInterface, String code, Boolean isPartial, PureModelContextData data) { try { @@ -127,7 +127,7 @@ public static CompletionResult getCodeTypeahead(String code, Boolean isPartial, String existingCode = func.body.get(0).accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build()); queryCode = existingCode + code; } - CompletionResult result = new Completer(graphCode, Lists.mutable.with(new RelationalCompleterExtension())).complete(queryCode); + CompletionResult result = new Completer(graphCode, Lists.mutable.with(new RelationalCompleterExtension()), legendInterface).complete(queryCode); if (result.getEngineException() != null) { return new CompletionResult(Lists.mutable.empty()); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryBuilder.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryBuilder.java index 21250f141cb..1ee74105a17 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryBuilder.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryBuilder.java @@ -140,7 +140,7 @@ public HttpHandler getHandler(REPLServerState state) String requestBody = bufferReader.lines().collect(Collectors.joining()); DataCubeQueryTypeaheadInput input = state.objectMapper.readValue(requestBody, DataCubeQueryTypeaheadInput.class); PureModelContextData data = state.getCurrentPureModelContextData(); - CompletionResult result = DataCubeHelpers.getCodeTypeahead(input.code, input.isPartial, data); + CompletionResult result = DataCubeHelpers.getCodeTypeahead(state.legendInterface, input.code, input.isPartial, data); handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result.getCompletion()), state); } catch (Exception e) diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestDataCubeHelpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestDataCubeHelpers.java index 39b51301e05..f36793d0dae 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestDataCubeHelpers.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestDataCubeHelpers.java @@ -214,7 +214,7 @@ private void testTypeahead(String expectedResult, String code, boolean isPartial { try { - Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getCodeTypeahead(code, isPartial, pureModelContextData))); + Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getCodeTypeahead(this.legendInterface, code, isPartial, pureModelContextData))); } catch (IOException e) { diff --git a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java index 8012083fca1..d5cc6968707 100644 --- a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java +++ b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java @@ -270,8 +270,7 @@ public void run(T serverConfiguration, Environment environment) DeploymentStateAndVersions.DEPLOYMENT_MODE = serverConfiguration.deployment.mode; SDLCLoader sdlcLoader = new SDLCLoader(serverConfiguration.metadataserver, null); - ForkJoinPool compilerPool = new ForkJoinPool(); - ModelManager modelManager = new ModelManager(serverConfiguration.deployment.mode, compilerPool, sdlcLoader); + ModelManager modelManager = new ModelManager(serverConfiguration.deployment.mode, sdlcLoader); ChainFixingFilterHandler.apply(environment.getApplicationContext(), serverConfiguration.filterPriorities);