diff --git a/legend-engine-config/legend-engine-repl/README.md b/legend-engine-config/legend-engine-repl/README.md new file mode 100644 index 00000000000..94a981e854f --- /dev/null +++ b/legend-engine-config/legend-engine-repl/README.md @@ -0,0 +1,72 @@ +# Legend REPL + +## Quick Start + +Run the REPL either in IDE (IntelliJ) or assembling a JAR and run the JAR. Within the REPL, to start + +> For autocomplete to work properly, it is recommended to run the REPL in a terminal; integrated terminal +> in IDE often override hotkeys / keybindings that are used by the REPL. _Developers are also recommended +> to work in a terminal to better test the interactions._ + +```sh +help # to see the list of commands + +# first load a CSV file into local DuckDB +load data.csv local::DuckDuckConnection test1 +# then show the data +#>{local::DuckDuckDatabase.test1}#->sort([])->from(local::DuckDuckRuntime) + +# to show the result grid in GUI mode +show +# to debug when error occurs, toggle debug mode +debug +``` + +## Configuration + +```shell +java \ + # Specify ag-grid license key for full enterprise functionalities support + -Dlegend.repl.dataCube.gridLicenseKey=YOUR_LICENSE_KEY \ + + # [DEVELOPMENT] Specify the base URL for the development instance of the web application + # this is needed to bypass CORS + -Dlegend.repl.dataCube.devWebAppBaseUrl=http://localhost:9005 \ + + # [DEVELOPMENT] By default, the port is randomized, but for development, the port needs + # to be fixed to allow the web application to connect to the REPL + -Dlegend.repl.dataCube.devPort=9006 \ + + -jar legend-engine-repl.jar +``` + +## Developer Guide + +### REPL Development Setup + +To debug the REPL, you can either run it in Debug mode in IntelliJ, which would compromise certain features, such as autocomplete +or you can run the REPL in a terminal. You can then setup a Remote JVM Debugger from IntelliJ to attach to the REPL instance. +The exact command to run it can be copied from the command run by IntelliJ (and removing some IntelliJ specifics from the classpath), for example: + +```shell +# NOTE: this command has a very long classpath! +java -Dfile.encoding=UTF-8 -classpath ... -Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5005 org.finos.legend.engine.repl.relational.client.RClient +``` + +Then, in IntelliJ, start the Remote JVM Debug session with the specified port (5005 like in command above). + +![img.png](docs/repl-debug-setup.png) + +### DataCube Development Setup + +Configure `legend.repl.dataCube.devWebAppBaseUrl` and `legend.repl.dataCube.devPort`. See screenshot below for an example of +how to configure the REPL in IntelliJ + +![img.png](docs/repl-webapp-dev-setup.png) + +Or if in a terminal + +```shell +# NOTE: this command has a very long classpath! +java -Dfile.encoding=UTF-8 -classpath ... -Dlegend.repl.devPort=9006 -Dlegend.repl.devWebAppBaseUrl=http://localhost:9005 org.finos.legend.engine.repl.relational.client.RClient +``` \ No newline at end of file diff --git a/legend-engine-config/legend-engine-repl/docs/repl-debug-setup.png b/legend-engine-config/legend-engine-repl/docs/repl-debug-setup.png new file mode 100644 index 00000000000..fc4c1b10910 Binary files /dev/null and b/legend-engine-config/legend-engine-repl/docs/repl-debug-setup.png differ diff --git a/legend-engine-config/legend-engine-repl/docs/repl-webapp-dev-setup.png b/legend-engine-config/legend-engine-repl/docs/repl-webapp-dev-setup.png new file mode 100644 index 00000000000..ef2616a63b6 Binary files /dev/null and b/legend-engine-config/legend-engine-repl/docs/repl-webapp-dev-setup.png differ diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java index e6b0525ef43..341e26b2685 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java @@ -60,6 +60,9 @@ import java.util.List; import java.util.Objects; +import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_QUALIFIED_PATH; +import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_SIGNATURE; + public class Completer { private final String buildCodeContext; @@ -82,7 +85,7 @@ public Completer(String buildCodeContext, MutableList extens buildCodeContext + "\n###Pure\n" + "import meta::pure::functions::relation::*;\n" + - "function _pierre::func():Any[*]{\n"; + "function " + REPL_RUN_FUNCTION_SIGNATURE + "{\n"; this.lineOffset = StringUtils.countMatches(header, "\n") + 1; this.handlers = Lists.mutable.with( new FilterHandler(), @@ -158,7 +161,7 @@ else if (topExpression instanceof AppliedFunction) if (currentExpression == topExpression) { // The top function name is being written, propose candidates - return new CompletionResult(getFunctionCandidates(leftCompiledVS, pureModel, null).select(c -> c.startsWith(currentlyTypeFunctionName)).collect(c -> new CompletionItem(c, c))); + return new CompletionResult(getFunctionCandidates(leftCompiledVS, pureModel, null).select(c -> c.startsWith(currentlyTypeFunctionName)).collect(c -> new CompletionItem(c, c + "("))); } else if (handler != null) { @@ -238,7 +241,7 @@ private ValueSpecification parseValueSpecification(String value) { String code = header + value + "\n" + "\n}"; PureModelContextData pureModelContextData = PureGrammarParser.newInstance().parseModel(code); - Function func = (Function) ListIterate.select(pureModelContextData.getElements(), s -> s.getPath().equals("_pierre::func__Any_MANY_")).getFirst(); + Function func = (Function) ListIterate.select(pureModelContextData.getElements(), s -> s.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH)).getFirst(); return func.body.get(0); } @@ -260,14 +263,14 @@ else if (leftType._rawType().getName().equals("String")) } else { - return Lists.mutable.with("count"); + return Lists.mutable.with("count", "joinStrings"); } } else if (org.finos.legend.pure.m3.navigation.type.Type.subTypeOf(leftType._rawType(), pureModel.getType(M3Paths.Number), pureModel.getExecutionSupport().getProcessorSupport())) { if (org.finos.legend.pure.m3.navigation.multiplicity.Multiplicity.isToOne(multiplicity)) { - return Lists.mutable.with("sqrt", "pow", "exp"); + return Lists.mutable.with("abs", "pow", "sqrt", "exp"); } else { diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java index 5a933d5a380..8d8f87172e5 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java @@ -24,21 +24,22 @@ import org.finos.legend.engine.repl.client.jline3.JLine3Parser; import org.finos.legend.engine.repl.core.Command; import org.finos.legend.engine.repl.core.ReplExtension; -import org.finos.legend.engine.repl.core.commands.Debug; -import org.finos.legend.engine.repl.core.commands.Execute; -import org.finos.legend.engine.repl.core.commands.Ext; -import org.finos.legend.engine.repl.core.commands.Graph; -import org.finos.legend.engine.repl.core.commands.Help; +import org.finos.legend.engine.repl.core.commands.*; import org.finos.legend.engine.repl.core.legend.LegendInterface; import org.finos.legend.engine.repl.core.legend.LocalLegendInterface; import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; +import org.jline.reader.EndOfFileException; import org.jline.reader.LineReader; import org.jline.reader.LineReaderBuilder; +import org.jline.reader.UserInterruptException; import org.jline.terminal.Terminal; import org.jline.terminal.TerminalBuilder; import org.jline.utils.AttributedStringBuilder; import org.jline.utils.AttributedStyle; +import static org.jline.jansi.Ansi.ansi; +import static org.jline.reader.LineReader.BLINK_MATCHING_PAREN; + public class Client { private final LegendInterface legendInterface = new LocalLegendInterface(); @@ -50,7 +51,6 @@ public class Client private ModelState state; private final PlanExecutor planExecutor; - public static void main(String[] args) throws Exception { new Client(Lists.mutable.empty(), Lists.mutable.empty(), PlanExecutor.newPlanExecutorBuilder().withAvailableStoreExecutors().build()).loop(); @@ -61,17 +61,14 @@ public static void main(String[] args) throws Exception public Client(MutableList replExtensions, MutableList completerExtensions, PlanExecutor planExecutor) throws Exception { this.replExtensions = replExtensions; - this.completerExtensions = completerExtensions; - this.planExecutor = planExecutor; - this.state = new ModelState(this.legendInterface, this.replExtensions); + this.terminal = TerminalBuilder.terminal(); replExtensions.forEach(e -> e.initialize(this)); - this.terminal = TerminalBuilder.terminal(); - + this.terminal.writer().println(ansi().fgBrightBlack().a("Welcome to the Legend REPL! Press 'Enter' or type 'help' to see the list of available commands.").reset()); this.terminal.writer().println("\n" + Logos.logos.get((int) (Logos.logos.size() * Math.random())) + "\n"); this.commands = replExtensions @@ -89,8 +86,17 @@ public Client(MutableList replExtensions, MutableList at the beginning of line will insert a tab instead of triggering a completion + // which will cause error since the completer doesn't handle such case + // See https://github.com/jline/jline3/wiki/Completion + .option(LineReader.Option.INSERT_TAB, true) + // Make sure word navigation works properly with Alt + (left/right) arrow key + .variable(LineReader.WORDCHARS, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-$") .highlighter(new JLine3Highlighter()) - .parser(new JLine3Parser())//new DefaultParser().quoteChars(new char[]{'"'})) + .parser(new JLine3Parser()) .completer(new JLine3Completer(this.commands)) .build(); @@ -105,16 +111,16 @@ public void loop() { while (true) { - String line = this.reader.readLine("> "); - if (line == null || line.equalsIgnoreCase("exit")) + try { - break; - } + String line = this.reader.readLine("> "); + if (line == null || line.equalsIgnoreCase("exit")) + { + break; + } - this.reader.getHistory().add(line); + this.reader.getHistory().add(line); - try - { this.commands.detect(new CheckedPredicate() { @Override @@ -126,14 +132,34 @@ public boolean safeAccept(Command c) throws Exception } catch (EngineException e) { - printError(e, line); + printError(e, this.reader.getBuffer().toString()); + } + // handle Ctrl + C: if the input is not empty, start a new line; otherwise, exit + catch (UserInterruptException e) + { + String lineContent = this.reader.getBuffer().toString(); + if (lineContent.isEmpty()) + { + System.exit(0); + break; + } + else + { + this.loop(); + } } - catch (Exception ee) + // handle Ctrl + D: exit + catch (EndOfFileException e) { - this.terminal.writer().println(ee.getMessage()); + System.exit(0); + break; + } + catch (Exception e) + { + this.terminal.writer().println(ansi().fgRed().a(e.getMessage()).reset()); if (this.debug) { - ee.printStackTrace(); + e.printStackTrace(); } } } @@ -204,4 +230,9 @@ public MutableList getCompleterExtensions() { return this.completerExtensions; } + + public Execute getExecuteCommand() + { + return (Execute) this.commands.detect(c -> c instanceof Execute); + } } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Parser.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Parser.java index 0fef62df797..1ad76691464 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Parser.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Parser.java @@ -45,7 +45,7 @@ public MyParsedLine(ParserResult result) public String word() { int index = wordIndex(); - if (result.words.size() > index) + if (index >= 0 && result.words.size() > index) { return result.words.get(index); } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Command.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Command.java index 3982a58ab54..e1dfa3e6875 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Command.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Command.java @@ -25,5 +25,10 @@ public interface Command public String documentation(); + public default String description() + { + return ""; + } + public MutableList complete(String cmd, LineReader lineReader, ParsedLine parsedLine); } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Helpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Helpers.java index fcae097b867..25edbc2a623 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Helpers.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Helpers.java @@ -20,6 +20,9 @@ public class Helpers { + public static final String REPL_RUN_FUNCTION_QUALIFIED_PATH = "repl::__internal__::run__Any_MANY_"; + public static final String REPL_RUN_FUNCTION_SIGNATURE = "repl::__internal__::run():Any[*]"; + public static Identity resolveIdentityFromLocalSubject(Client client) { try diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/ReplExtension.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/ReplExtension.java index 8a2a58f70bd..7717197fd6a 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/ReplExtension.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/ReplExtension.java @@ -42,8 +42,6 @@ default MutableList typeGroup() MutableList getExtraCommands(); -// MutableList getExtraState(); - boolean supports(Result res); String print(Result res); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Debug.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Debug.java index a3caf0a0081..fb44d8503f6 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Debug.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Debug.java @@ -36,6 +36,12 @@ public String documentation() return "debug ()"; } + @Override + public String description() + { + return "toggle debug mode"; + } + @Override public boolean process(String line) throws Exception { diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Execute.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Execute.java index e00fc880553..a7584b33698 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Execute.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Execute.java @@ -44,14 +44,14 @@ import java.util.HashMap; +import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_SIGNATURE; + public class Execute implements Command { private static final ObjectMapper objectMapper = new ObjectMapper(); private final Client client; - private final PlanExecutor planExecutor; - - private PureModelContextData currentPMCD; + private ExecuteResult lastExecuteResult; public Execute(Client client, PlanExecutor planExecutor) { @@ -59,9 +59,9 @@ public Execute(Client client, PlanExecutor planExecutor) this.planExecutor = planExecutor; } - public PureModelContextData getCurrentPMCD() + public ExecuteResult getLastExecuteResult() { - return currentPMCD; + return this.lastExecuteResult; } @Override @@ -115,10 +115,9 @@ private Candidate buildCandidate(CompletionItem s) public String execute(String txt) { String code = "###Pure\n" + - "function a::b::c::d():Any[*]\n{\n" + txt + ";\n}"; + "function " + REPL_RUN_FUNCTION_SIGNATURE + "\n{\n" + txt + ";\n}"; PureModelContextData d = this.client.getModelState().parseWithTransient(code); - this.currentPMCD = d; if (this.client.isDebug()) { @@ -150,7 +149,13 @@ public String execute(String txt) // Execute Identity identity = Helpers.resolveIdentityFromLocalSubject(this.client); - Result res = this.planExecutor.execute((SingleExecutionPlan) PlanExecutor.readExecutionPlan(planStr), new HashMap<>(), identity.getName(), identity, null); + SingleExecutionPlan execPlan = (SingleExecutionPlan) PlanExecutor.readExecutionPlan(planStr); + Result res = this.planExecutor.execute(execPlan, new HashMap<>(), identity.getName(), identity, null); + + // Store these infos for commands that need to access data from the latest execute + this.lastExecuteResult = new ExecuteResult(d, pureModel, res, execPlan); + + // Show result if (res instanceof ConstantResult) { return ((ConstantResult) res).getValue().toString(); @@ -173,4 +178,20 @@ public PlanExecutor getPlanExecutor() { return this.planExecutor; } + + public static class ExecuteResult + { + public final PureModelContextData pureModelContextData; + public final PureModel pureModel; + public final Result result; + public final SingleExecutionPlan plan; + + public ExecuteResult(PureModelContextData pureModelContextData, PureModel pureModel, Result result, SingleExecutionPlan plan) + { + this.pureModelContextData = pureModelContextData; + this.pureModel = pureModel; + this.result = result; + this.plan = plan; + } + } } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Ext.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Ext.java index 3d87eb7a7c7..b8c0ecf7330 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Ext.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Ext.java @@ -39,6 +39,12 @@ public String documentation() return "ext"; } + @Override + public String description() + { + return "show loaded extensions"; + } + @Override public boolean process(String line) throws Exception { diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Graph.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Graph.java index 2e3315bae17..4939186aa09 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Graph.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Graph.java @@ -45,7 +45,13 @@ public Graph(Client client) @Override public String documentation() { - return "graph ()"; + return "graph ()"; + } + + @Override + public String description() + { + return "show graph element definition in Pure"; } @Override diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Help.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Help.java index 0811ab5cf94..5c398163436 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Help.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Help.java @@ -21,6 +21,9 @@ import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; +import java.util.Collections; +import java.util.Comparator; + public class Help implements Command { private final MutableList commands; @@ -38,12 +41,23 @@ public String documentation() return "help"; } + @Override + public String description() + { + return "show available commands and their usage"; + } + @Override public boolean process(String cmd) throws Exception { if (cmd.isEmpty() || cmd.equals("help")) { - this.client.getTerminal().writer().println(this.commands.collect(c -> " " + c.documentation()).makeString("\n")); + int maxDocLength = this.commands.maxBy(c -> c.documentation().length()).documentation().length(); + this.client.getTerminal().writer().println(this.commands + .sortThis(Comparator.comparing(Command::documentation)) + // pad right to align the command description + .collect(c -> " " + c.documentation() + String.join("", Collections.nCopies(maxDocLength - c.documentation().length() + 2, " ")) + c.description()) + .makeString("\n")); return true; } return false; diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java index 42bc61c61af..a14a2c8afbd 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java @@ -30,6 +30,8 @@ import java.net.URL; +import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_QUALIFIED_PATH; + public class LocalLegendInterface implements LegendInterface { @Override @@ -82,7 +84,7 @@ public PureModel compile(PureModelContextData pureModelContextData) public Root_meta_pure_executionPlan_ExecutionPlan generatePlan(PureModel pureModel, boolean debug) { RichIterable extensions = PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport())); - Pair res = PlanGenerator.generateExecutionPlanAsPure(pureModel.getConcreteFunctionDefinition_safe("a::b::c::d__Any_MANY_"), null, pureModel, PlanPlatform.JAVA, "", debug, extensions); + Pair res = PlanGenerator.generateExecutionPlanAsPure(pureModel.getConcreteFunctionDefinition_safe(REPL_RUN_FUNCTION_QUALIFIED_PATH), null, pureModel, PlanPlatform.JAVA, "", debug, extensions); if (debug) { System.out.println(res.getTwo()); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/test/java/org/finos/legend/engine/repl/TestCompleter.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/test/java/org/finos/legend/engine/repl/TestCompleter.java index 2f0ca46a6f6..f96c88df161 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/test/java/org/finos/legend/engine/repl/TestCompleter.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/test/java/org/finos/legend/engine/repl/TestCompleter.java @@ -24,15 +24,15 @@ public class TestCompleter @Test public void testPrimitives() { - Assert.assertEquals("[sqrt , sqrt], [pow , pow], [exp , exp]", checkResultNoException(new Completer("").complete("1->"))); - Assert.assertEquals("[contains , contains], [startsWith , startsWith], [endsWith , endsWith], [toLower , toLower], [toUpper , toUpper], [lpad , lpad], [rpad , rpad], [parseInteger , parseInteger], [parseFloat , parseFloat]", checkResultNoException(new Completer("").complete("'a'->"))); - Assert.assertEquals("[sum , sum], [mean , mean], [average , average], [min , min], [max , max], [count , count], [percentile , percentile], [variancePopulation , variancePopulation], [varianceSample , varianceSample], [stdDevPopulation , stdDevPopulation], [stdDevSample , stdDevSample]", checkResultNoException(new Completer("").complete("[1,2]->"))); + Assert.assertEquals("[abs , abs(], [pow , pow(], [sqrt , sqrt(], [exp , exp(]", checkResultNoException(new Completer("").complete("1->"))); + Assert.assertEquals("[contains , contains(], [startsWith , startsWith(], [endsWith , endsWith(], [toLower , toLower(], [toUpper , toUpper(], [lpad , lpad(], [rpad , rpad(], [parseInteger , parseInteger(], [parseFloat , parseFloat(]", checkResultNoException(new Completer("").complete("'a'->"))); + Assert.assertEquals("[sum , sum(], [mean , mean(], [average , average(], [min , min(], [max , max(], [count , count(], [percentile , percentile(], [variancePopulation , variancePopulation(], [varianceSample , varianceSample(], [stdDevPopulation , stdDevPopulation(], [stdDevSample , stdDevSample(]", checkResultNoException(new Completer("").complete("[1,2]->"))); } @Test public void testArrowOnType() { - Assert.assertEquals("[project , project]", checkResultNoException(new Completer("Class x::A{name:String[1];other:Integer[1];}").complete("x::A.all()->"))); + Assert.assertEquals("[project , project(]", checkResultNoException(new Completer("Class x::A{name:String[1];other:Integer[1];}").complete("x::A.all()->"))); Assert.assertEquals("", checkResultNoException(new Completer("Class x::A{name:String[1];other:Integer[1];}").complete("x::A.all()->fu"))); } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/pom.xml b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/pom.xml index 2c25c30576c..34c572c680f 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/pom.xml +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/pom.xml @@ -179,6 +179,10 @@ org.finos.legend.engine legend-engine-identity-core + + org.finos.legend.engine + legend-engine-xt-identity-kerberos + org.jline @@ -199,11 +203,14 @@ eclipse-collections-api + + com.fasterxml.jackson.core + jackson-core + com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.core jackson-annotations @@ -219,4 +226,126 @@ jline + + + + repl-dev + + true + + + + org.slf4j + slf4j-nop + runtime + ${slf4j.version} + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-web-content + process-resources + + copy-resources + + + ${project.build.directory}/classes/web-content + + + ${project.basedir}/../../../temp/web-content + + + + + + + + maven-shade-plugin + + + package + + shade + + + true + false + + + *:* + + module-info.class + META-INF/**/module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + org.finos.legend.engine.repl.relational.client.RClient + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + default + + enforce + + + + + ${maven.enforcer.requireJavaVersion} + + + ${maven.enforcer.requireMavenVersion} + + + + + + log4j:*:*:*:compile + log4j:*:*:*:runtime + org.slf4j:*:*:*:compile + org.slf4j:*:*:*:runtime + commons-logging + javax.mail + + + + + org.slf4j:slf4j-nop:${slf4j.version} + org.slf4j:jul-to-slf4j:${slf4j.version} + org.slf4j:slf4j-api:${slf4j.version} + org.slf4j:jcl-over-slf4j:${slf4j.version} + org.slf4j:slf4j-jdk14:${slf4j.version} + org.slf4j:slf4j-log4j12:${slf4j.version} + org.slf4j:slf4j-ext:${slf4j.version} + log4j:log4j:${log4j.version} + log4j:apache-log4j-extras:${log4j.version} + + + + + + + + + + + diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/RelationalReplExtension.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/RelationalReplExtension.java index 8c7646ade1f..72738eb14ed 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/RelationalReplExtension.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/RelationalReplExtension.java @@ -22,14 +22,11 @@ import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.core.Command; import org.finos.legend.engine.repl.core.ReplExtension; -import org.finos.legend.engine.repl.relational.commands.Cache; -import org.finos.legend.engine.repl.relational.commands.DB; -import org.finos.legend.engine.repl.relational.commands.Load; +import org.finos.legend.engine.repl.relational.commands.*; import org.finos.legend.engine.repl.relational.local.LocalConnectionManagement; import org.finos.legend.engine.repl.relational.local.LocalConnectionType; -import org.finos.legend.engine.repl.relational.commands.Show; -import org.finos.legend.engine.repl.relational.httpServer.ReplGridServer; +import org.finos.legend.engine.repl.relational.server.REPLServer; import java.awt.*; import java.sql.SQLException; @@ -39,7 +36,7 @@ public class RelationalReplExtension implements ReplExtension { private Client client; - public ReplGridServer replGridServer; + public REPLServer REPLServer; private LocalConnectionManagement localConnectionManagement; @@ -77,8 +74,8 @@ public void initialize(Client client) try { - this.replGridServer = new ReplGridServer(this.client); - this.replGridServer.initializeServer(); + this.REPLServer = new REPLServer(this.client); + this.REPLServer.initialize(); } catch (Exception e) { @@ -95,10 +92,13 @@ public MutableList generateDynamicContent(String code) @Override public MutableList getExtraCommands() { - MutableList extraCommands = Lists.mutable.with(new DB(this.client, this), new Load(this.client, this)); - extraCommands.add(new Show(this.client, this.replGridServer)); - extraCommands.add(new Cache(this.client, this.client.getPlanExecutor())); - return extraCommands; + return Lists.mutable.with( + new DB(this.client, this), + new Load(this.client, this), + new Drop(this.client), + new Show(this.client, this.REPLServer), + new Cache(this.client, this.client.getPlanExecutor()) + ); } @Override diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/autocomplete/RelationalCompleterExtension.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/autocomplete/RelationalCompleterExtension.java index 188d580528b..2f220470bf0 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/autocomplete/RelationalCompleterExtension.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/autocomplete/RelationalCompleterExtension.java @@ -62,6 +62,10 @@ public CompletionResult extraClassInstanceProcessor(Object islandExpr, PureModel private static boolean nameMatch(PackageableElement c, String writtenPath) { String path = org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c); + if (path.isEmpty()) // NOTE: handle an edge case where stub store is added to the graph + { + return false; + } if (path.length() > writtenPath.length()) { return path.startsWith(writtenPath); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java index f4d34b90d5d..b72f3dfd458 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java @@ -41,9 +41,9 @@ import org.finos.legend.engine.repl.core.Helpers; import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; import org.finos.legend.engine.shared.core.identity.Identity; -import org.finos.legend.pure.generated.core_pure_executionPlan_executionPlan_print; import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan; import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; +import org.finos.legend.pure.generated.core_pure_executionPlan_executionPlan_print; import org.jline.reader.Candidate; import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; @@ -54,6 +54,7 @@ import java.sql.Statement; import java.util.HashMap; +import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_SIGNATURE; import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; public class Cache implements Command @@ -73,6 +74,12 @@ public String documentation() return "cache "; } + @Override + public String description() + { + return "cache the result of the last executed query into a table"; + } + @Override public boolean process(String line) throws Exception { @@ -90,7 +97,7 @@ public boolean process(String line) throws Exception DatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), connectionPath); String code = "###Pure\n" + - "function a::b::c::d():Any[*]\n{\n" + expression + ";\n}"; + "function " + REPL_RUN_FUNCTION_SIGNATURE + "\n{\n" + expression + ";\n}"; PureModelContextData parsed = this.client.getModelState().parseWithTransient(code); PureModel pureModel = this.client.getLegendInterface().compile(parsed); RichIterable extensions = PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport())); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/DB.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/DB.java index dab9a8c902c..0135bb4a691 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/DB.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/DB.java @@ -50,6 +50,12 @@ public String documentation() return "db "; } + @Override + public String description() + { + return "show schema summary of the database"; + } + @Override public boolean process(String line) throws Exception { diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Drop.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Drop.java new file mode 100644 index 00000000000..4d332313748 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Drop.java @@ -0,0 +1,122 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.commands; + +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.factory.Lists; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerUtility; +import org.finos.legend.engine.plan.execution.stores.relational.connection.driver.DatabaseManager; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.repl.client.Client; +import org.finos.legend.engine.repl.core.Command; +import org.finos.legend.engine.repl.relational.RelationalReplExtension; +import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; +import org.jline.builtins.Completers; +import org.jline.reader.Candidate; +import org.jline.reader.LineReader; +import org.jline.reader.ParsedLine; + +import java.io.File; +import java.sql.Connection; +import java.sql.Statement; + +import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; + +public class Drop implements Command +{ + private final Client client; + private final Completers.FilesCompleter completer = new Completers.FilesCompleter(new File("/")); + + public Drop(Client client) + { + this.client = client; + } + + @Override + public String documentation() + { + return "drop "; + } + + @Override + public boolean process(String line) throws Exception + { + if (line.startsWith("drop")) + { + String[] tokens = line.split(" "); + if (tokens.length != 3) + { + throw new RuntimeException("Error, drop should be used as '" + this.documentation() + "'"); + } + + DatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), tokens[1]); + + try (Connection connection = ConnectionHelper.getConnection(databaseConnection, client.getPlanExecutor())) + { + String tableName = tokens[2]; + try (Statement statement = connection.createStatement()) + { + statement.executeUpdate(DatabaseManager.fromString(databaseConnection.type.name()).relationalDatabaseSupport().dropTable(tableName, tokens[1])); + this.client.getTerminal().writer().println("Dropped table: '" + tableName + "'"); + } + } + + return true; + } + return false; + } + + @Override + public MutableList complete(String inScope, LineReader lineReader, ParsedLine parsedLine) + { + if (inScope.startsWith("drop ")) + { + MutableList words = Lists.mutable.withAll(parsedLine.words()).drop(2); + if (!words.contains(" ")) + { + String start = words.get(0); + PureModelContextData d = this.client.getModelState().parse(); + return ListIterate.select(d.getElementsOfType(PackageableConnection.class), c -> !c._package.equals("__internal__")) + .collect(c -> PureGrammarComposerUtility.convertPath(c.getPath())) + .select(c -> c.startsWith(start)) + .collect(Candidate::new); + } + else + { + String connectionPath = words.subList(0, words.indexOf(" ") + 1).makeString("").trim(); + String start = words.subList(words.indexOf(" ") + 1, words.size()).get(0); + PureModelContextData d = this.client.getModelState().parse(); + MutableList foundConnections = ListIterate.select(d.getElementsOfType(PackageableConnection.class), c -> !c._package.equals("__internal__")) + .select(c -> PureGrammarComposerUtility.convertPath(c.getPath()).equals(connectionPath)); + if (!foundConnections.isEmpty() && foundConnections.getFirst().connectionValue instanceof DatabaseConnection) + { + try (Connection connection = ConnectionHelper.getConnection((DatabaseConnection) foundConnections.getFirst().connectionValue, client.getPlanExecutor())) + { + return getTables(connection).select(c -> c.name.startsWith(start)).collect(c -> c.name).collect(Candidate::new); + } + catch (Exception e) + { + // do nothing + } + } + return null; + } + } + return null; + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Load.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Load.java index 52bb911829b..b711df2c34b 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Load.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Load.java @@ -53,7 +53,13 @@ public Load(Client client, RelationalReplExtension relationalReplExtension) @Override public String documentation() { - return "load "; + return "load (
)"; + } + + @Override + public String description() + { + return "load CSV file into table"; } @Override @@ -62,16 +68,16 @@ public boolean process(String line) throws Exception if (line.startsWith("load")) { String[] tokens = line.split(" "); - if (tokens.length != 3) + if (tokens.length != 3 && tokens.length != 4) { - throw new RuntimeException("Error, load should be used as 'load '"); + throw new RuntimeException("Error, load should be used as '" + this.documentation() + "'"); } DatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), tokens[2]); try (Connection connection = ConnectionHelper.getConnection(databaseConnection, client.getPlanExecutor())) { - String tableName = "test" + (getTables(connection).size() + 1); + String tableName = tokens.length == 4 ? tokens[3] : ("test" + (getTables(connection).size() + 1)); try (Statement statement = connection.createStatement()) { diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Show.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Show.java index d0cb0cf3397..3fea57f989c 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Show.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Show.java @@ -15,28 +15,29 @@ package org.finos.legend.engine.repl.relational.commands; import org.eclipse.collections.api.list.MutableList; -import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.core.Command; import org.finos.legend.engine.repl.core.commands.Execute; -import org.finos.legend.engine.repl.relational.httpServer.ReplGridServer; +import org.finos.legend.engine.repl.relational.server.REPLServer; import org.jline.reader.Candidate; import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; -import java.awt.Desktop; +import java.awt.*; import java.net.URI; +import static org.jline.jansi.Ansi.ansi; + public class Show implements Command { - private Client client; + private final Client client; - public ReplGridServer replGridServer; + public REPLServer REPLServer; - public Show(Client client, ReplGridServer replGridServer) + public Show(Client client, REPLServer REPLServer) { this.client = client; - this.replGridServer = replGridServer; + this.REPLServer = REPLServer; } @Override @@ -45,33 +46,39 @@ public String documentation() return "show"; } + @Override + public String description() + { + return "show the result for the last executed query in GUI mode (DataCube)"; + } + @Override public boolean process(String line) { if (line.startsWith("show")) { - PureModelContextData currentPMCD = ((Execute) this.client.commands.getLast()).getCurrentPMCD(); - if (currentPMCD == null) + Execute.ExecuteResult lastExecuteResult = this.client.getExecuteCommand().getLastExecuteResult(); + if (lastExecuteResult == null) { - this.client.getTerminal().writer().println("Unable to show repl grid, no query has been executed"); + this.client.getTerminal().writer().println("Can't show result grid in DataCube. Try to run a query in REPL first..."); } else { + this.REPLServer.setExecuteResult(lastExecuteResult); try { - this.replGridServer.updateGridState(currentPMCD); - if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) - { - Desktop.getDesktop().browse(URI.create(replGridServer.getGridUrl())); - } - else - { - this.client.getTerminal().writer().println(replGridServer.getGridUrl()); - } + if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) + { + Desktop.getDesktop().browse(URI.create(REPLServer.getUrl())); + } + else + { + this.client.getTerminal().writer().println(REPLServer.getUrl()); + } } catch (Exception e) { - this.client.getTerminal().writer().println(e.getMessage()); + this.client.getTerminal().writer().println(ansi().fgRed().a(e.getMessage()).reset()); } } return true; diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/httpServer/ReplGridServer.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/httpServer/ReplGridServer.java deleted file mode 100644 index a26fda2c0a7..00000000000 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/httpServer/ReplGridServer.java +++ /dev/null @@ -1,421 +0,0 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.engine.repl.relational.httpServer; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; -import com.sun.net.httpserver.HttpServer; -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.commons.io.IOUtils; -import org.eclipse.collections.api.RichIterable; -import org.eclipse.collections.api.factory.Lists; -import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; -import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser; -import org.finos.legend.engine.language.pure.grammar.to.DEPRECATED_PureGrammarComposerCore; -import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposer; -import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerContext; -import org.finos.legend.engine.plan.execution.PlanExecutor; -import org.finos.legend.engine.plan.execution.result.Result; -import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat; -import org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult; -import org.finos.legend.engine.plan.generation.PlanGenerator; -import org.finos.legend.engine.plan.generation.transformers.LegendPlanTransformers; -import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.application.AppliedFunction; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.CInteger; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; -import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; -import org.finos.legend.engine.repl.autocomplete.Completer; -import org.finos.legend.engine.repl.autocomplete.CompletionResult; -import org.finos.legend.engine.repl.client.Client; -import org.finos.legend.engine.repl.core.legend.LegendInterface; -import org.finos.legend.engine.repl.relational.autocomplete.RelationalCompleterExtension; -import org.finos.legend.engine.shared.core.api.grammar.RenderStyle; -import org.finos.legend.engine.shared.core.operational.errorManagement.ExceptionError; -import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan; -import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; - -public class ReplGridServer -{ - private static final ObjectMapper objectMapper = new ObjectMapper(); - private static final PlanExecutor planExecutor = PlanExecutor.newPlanExecutorBuilder().withAvailableStoreExecutors().build(); - private PureModelContextData currentPMCD; - private final Client client; - private int port; - - public ReplGridServer(Client client) - { - this.client = client; - } - - public String getGridUrl() - { - return "http://localhost:" + this.port + "/repl/grid"; - } - - public static class GridServerResult - { - private final String currentQuery; - private final String result; - - public GridServerResult(@JsonProperty("currentQuery") String currentQuery, @JsonProperty("result") String result) - { - this.currentQuery = currentQuery; - this.result = result; - } - - public String getResult() - { - return this.result; - } - - public String getCurrentQuery() - { - return this.currentQuery; - } - } - - public void updateGridState(PureModelContextData pmcd) - { - this.currentPMCD = pmcd; - } - - public void initializeServer() throws Exception - { - InetSocketAddress serverPortAddress = new InetSocketAddress(0); - HttpServer server = HttpServer.create(serverPortAddress, 0); - - server.createContext("/licenseKey", new HttpHandler() - { - @Override - public void handle(HttpExchange exchange) throws IOException - { - if ("GET".equals(exchange.getRequestMethod())) - { - try - { - String licenseKey = System.getProperty("legend.repl.grid.licenseKey") == null ? "" : System.getProperty("legend.repl.grid.licenseKey"); - String key = objectMapper.writeValueAsString(licenseKey); - handleResponse(exchange, 200, key); - } - catch (Exception e) - { - OutputStream os = exchange.getResponseBody(); - exchange.sendResponseHeaders(500, e.getMessage().length()); - os.write(e.getMessage().getBytes(StandardCharsets.UTF_8)); - os.close(); - } - - } - } - }); - - server.createContext("/repl/", exchange -> - { - if ("GET".equals(exchange.getRequestMethod())) - { - String[] path = exchange.getRequestURI().getPath().split("/repl/"); - String resourcePath = "/web-content/package/dist/repl/" + (path[1].equals("grid") ? "index.html" : path[1]); - try (OutputStream os = exchange.getResponseBody(); - InputStream is = ReplGridServer.class.getResourceAsStream(resourcePath) - ) - { - if (is == null) - { - exchange.sendResponseHeaders(404, -1); - } - else - { - if (resourcePath.endsWith(".html")) - { - exchange.getResponseHeaders().add("Content-Type", "text/html; charset=utf-8"); - } - if (resourcePath.endsWith(".js")) - { - exchange.getResponseHeaders().add("Content-Type", "text/javascript; charset=utf-8"); - } - else if (resourcePath.endsWith(".css")) - { - exchange.getResponseHeaders().add("Content-Type", "text/css; charset=utf-8"); - } - - exchange.sendResponseHeaders(200, 0); - IOUtils.copy(is, os); - } - } - catch (Exception e) - { - handleResponse(exchange, 500, e.getMessage()); - } - } - }); - - server.createContext("/initialLambda", exchange -> - { - if ("GET".equals(exchange.getRequestMethod())) - { - try - { - Function func = (Function) currentPMCD.getElements().stream().filter(e -> e.getPath().equals("a::b::c::d__Any_MANY_")).collect(Collectors.toList()).get(0); - Lambda lambda = new Lambda(); - lambda.body = func.body; - String response = objectMapper.writeValueAsString(lambda); - handleResponse(exchange, 200, response); - } - catch (Exception e) - { - handleResponse(exchange, 500, e.getMessage()); - } - } - }); - - server.createContext("/executeLambda", exchange -> - { - if ("POST".equals(exchange.getRequestMethod())) - { - try - { - InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); - BufferedReader bufferReader = new BufferedReader(inputStreamReader); - String requestBody = bufferReader.lines().collect(Collectors.joining()); - AppliedFunction body = (AppliedFunction) PureGrammarParser.newInstance().parseValueSpecification(requestBody, "", 0, 0, true); - List newBody = Lists.mutable.of(body); - if (checkIfPaginationIsEnabled(exchange.getRequestURI().getQuery())) - { - applySliceFunction(newBody); - } - Function func = (Function) currentPMCD.getElements().stream().filter(e -> e.getPath().equals("a::b::c::d__Any_MANY_")).collect(Collectors.toList()).get(0); - func.body = newBody; - String response = executeLambda(client.getLegendInterface(), currentPMCD, func, newBody.get(0)); - handleResponse(exchange, 200, response); - } - catch (Exception e) - { - handleResponse(exchange, 500, e.getMessage()); - } - } - }); - - server.createContext("/typeahead", exchange -> - { - if ("POST".equals(exchange.getRequestMethod())) - { - try - { - InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); - BufferedReader bufferReader = new BufferedReader(inputStreamReader); - String requestBody = bufferReader.lines().collect(Collectors.joining()); - String buildCodeContext = PureGrammarComposer.newInstance(PureGrammarComposerContext.Builder.newInstance().build()).renderPureModelContextData(currentPMCD); - CompletionResult result = new Completer(buildCodeContext, Lists.mutable.with(new RelationalCompleterExtension())).complete(requestBody); - if (result.getEngineException() != null) - { - handleResponse(exchange, 500, result.getEngineException().toPretty()); - } - else - { - handleResponse(exchange, 200, objectMapper.writeValueAsString(result.getCompletion())); - } - } - catch (Exception e) - { - handleResponse(exchange, 500, e.getMessage()); - } - } - }); - - server.createContext("/parseQuery", exchange -> - { - if ("POST".equals(exchange.getRequestMethod())) - { - try - { - InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); - BufferedReader bufferReader = new BufferedReader(inputStreamReader); - String requestBody = bufferReader.lines().collect(Collectors.joining("\n")); - PureGrammarParser.newInstance().parseValueSpecification(requestBody, "", 0, 0, true); - exchange.sendResponseHeaders(200, -1); - } - catch (Exception e) - { - handleResponse(exchange, 400, objectMapper.writeValueAsString(new ExceptionError(-1, e))); - } - } - }); - - server.createContext("/gridResult", exchange -> - { - if ("GET".equals(exchange.getRequestMethod())) - { - ValueSpecification funcBody = null; - Function func = null; - try - { - func = (Function) currentPMCD.getElements().stream().filter(e -> e.getPath().equals("a::b::c::d__Any_MANY_")).collect(Collectors.toList()).get(0); - funcBody = func.body.get(0); - - List newBody = Lists.mutable.of(funcBody); - if (checkIfPaginationIsEnabled(exchange.getRequestURI().getQuery())) - { - applySliceFunction(newBody); - } - func.body = newBody; - - String response = executeLambda(client.getLegendInterface(), currentPMCD, func, funcBody); - handleResponse(exchange, 200, response); - } - catch (Exception e) - { - System.out.println(e.getMessage()); - if (func != null) - { - func.body = Lists.mutable.of(funcBody); - } - handleResponse(exchange, 500, e.getMessage()); - } - - } - else if ("POST".equals(exchange.getRequestMethod())) - { - ValueSpecification funcBody = null; - Function func = null; - try - { - InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); - BufferedReader bufferReader = new BufferedReader(inputStreamReader); - String requestBody = bufferReader.lines().collect(Collectors.joining()); - Lambda request = objectMapper.readValue(requestBody, Lambda.class); - func = (Function) currentPMCD.getElements().stream().filter(e -> e.getPath().equals("a::b::c::d__Any_MANY_")).collect(Collectors.toList()).get(0); - funcBody = func.body.get(0); - func.body = request.body; - String response = executeLambda(client.getLegendInterface(), currentPMCD, func, funcBody); - handleResponse(exchange, 200, response); - } - catch (Exception e) - { - System.out.println(e.getMessage()); - if (func != null) - { - func.body = Lists.mutable.of(funcBody); - } - handleResponse(exchange, 500, e.getMessage()); - } - } - }); - - server.setExecutor(null); - server.start(); - this.port = server.getAddress().getPort(); - System.out.println("REPL Grid Server has started at port " + this.port); - } - - public static String executeLambda(LegendInterface legendInterface, PureModelContextData currentRequestPMCD, Function func, ValueSpecification funcBody) throws IOException - { - Lambda lambda = new Lambda(); - lambda.body = func.body; - String lambdaString = lambda.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().withRenderStyle(RenderStyle.PRETTY).build()); - PureModel pureModel = legendInterface.compile(currentRequestPMCD); - RichIterable extensions = PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport())); - - // Plan - Root_meta_pure_executionPlan_ExecutionPlan plan = legendInterface.generatePlan(pureModel, false); - String planStr = PlanGenerator.serializeToJSON(plan, "vX_X_X", pureModel, extensions, LegendPlanTransformers.transformers); - - // Execute - Result res = planExecutor.execute(planStr); - func.body = Lists.mutable.of(funcBody); - if (res instanceof RelationalResult) - { - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - ((RelationalResult) res).getSerializer(SerializationFormat.DEFAULT).stream(byteArrayOutputStream); - GridServerResult result = new GridServerResult(lambdaString, byteArrayOutputStream.toString()); - return objectMapper.writeValueAsString(result); - } - throw new RuntimeException("Expected return type of Lambda execution is RelationalResult, but returned " + res.getClass().getName()); - } - - private void handleResponse(HttpExchange exchange, int responseCode, String response) - { - try - { - OutputStream os = exchange.getResponseBody(); - byte[] byteResponse = response.getBytes(StandardCharsets.UTF_8); - exchange.sendResponseHeaders(responseCode, byteResponse.length); - os.write(byteResponse); - os.close(); - } - catch (IOException e) - { - System.out.println(e.getMessage()); - } - } - - public static void applySliceFunction(List body) - { - CInteger startValue = new CInteger(0); - CInteger endValue = new CInteger(100); - ValueSpecification currentExpression = body.get(0); - while (currentExpression instanceof AppliedFunction) - { - if (((AppliedFunction) currentExpression).function.equals("from")) - { - ValueSpecification childExpression = ((AppliedFunction) currentExpression).parameters.get(0); - if (childExpression instanceof AppliedFunction && ((AppliedFunction) childExpression).function.equals("slice")) - { - ((AppliedFunction) childExpression).parameters = Lists.mutable.of(((AppliedFunction) childExpression).parameters.get(0), startValue, endValue); - break; - } - AppliedFunction sliceFunction = new AppliedFunction(); - sliceFunction.function = "slice"; - sliceFunction.parameters = Lists.mutable.of(((AppliedFunction) currentExpression).parameters.get(0), startValue, endValue); - ((AppliedFunction) currentExpression).parameters.set(0, sliceFunction); - break; - } - currentExpression = ((AppliedFunction) currentExpression).parameters.get(0); - } - } - - private static boolean checkIfPaginationIsEnabled(String queryParamsString) - { - Map queryParams = new HashMap<>(); - for (String param : queryParamsString.split("&")) - { - String[] entry = param.split("="); - if (entry.length > 1) - { - queryParams.put(entry[0], entry[1]); - } - else - { - queryParams.put(entry[0], ""); - } - } - return queryParams.get("isPaginationEnabled").equals("true"); - } -} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/DataCubeHelpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/DataCubeHelpers.java new file mode 100644 index 00000000000..090a050a8f9 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/DataCubeHelpers.java @@ -0,0 +1,166 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server; + +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.tuple.Pair; +import org.eclipse.collections.impl.tuple.Tuples; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.RelationTypeHelper; +import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser; +import org.finos.legend.engine.language.pure.grammar.to.DEPRECATED_PureGrammarComposerCore; +import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposer; +import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerContext; +import org.finos.legend.engine.plan.execution.PlanExecutor; +import org.finos.legend.engine.plan.execution.result.Result; +import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat; +import org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult; +import org.finos.legend.engine.plan.generation.PlanGenerator; +import org.finos.legend.engine.plan.generation.transformers.LegendPlanTransformers; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function; +import org.finos.legend.engine.protocol.pure.v1.model.relationType.RelationType; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; +import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; +import org.finos.legend.engine.repl.autocomplete.Completer; +import org.finos.legend.engine.repl.autocomplete.CompletionResult; +import org.finos.legend.engine.repl.core.legend.LegendInterface; +import org.finos.legend.engine.repl.relational.autocomplete.RelationalCompleterExtension; +import org.finos.legend.engine.repl.relational.server.model.DataCubeExecutionResult; +import org.finos.legend.engine.shared.core.api.grammar.RenderStyle; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.finos.legend.engine.shared.core.kerberos.SubjectTools; +import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan; +import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; + +import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_QUALIFIED_PATH; + +public class DataCubeHelpers +{ + public static DataCubeExecutionResult executeQuery(LegendInterface legendInterface, PlanExecutor planExecutor, PureModelContextData data) throws IOException + { + PureModel pureModel = legendInterface.compile(data); + RichIterable extensions = PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport())); + + // Plan + Root_meta_pure_executionPlan_ExecutionPlan _plan = legendInterface.generatePlan(pureModel, false); + String planStr = PlanGenerator.serializeToJSON(_plan, "vX_X_X", pureModel, extensions, LegendPlanTransformers.transformers); + + // Execute + Identity identity; + try + { + identity = Identity.makeIdentity(SubjectTools.getLocalSubject()); + } + catch (Exception e) + { + // Can't resolve identity from local subject + identity = Identity.getAnonymousIdentity(); + } + + SingleExecutionPlan plan = (SingleExecutionPlan) PlanExecutor.readExecutionPlan(planStr); + + try (Result execResult = planExecutor.execute(plan, new HashMap<>(), identity.getName(), identity, null)) + { + if (execResult instanceof RelationalResult) + { + DataCubeExecutionResult result = new DataCubeExecutionResult(); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + ((RelationalResult) execResult).getSerializer(SerializationFormat.DEFAULT).stream(byteArrayOutputStream); + result.result = byteArrayOutputStream.toString(); + return result; + } + throw new RuntimeException("Expected execution result of type 'RelationalResult', but got '" + execResult.getClass().getName() + "'"); + } + } + + public static RelationType getRelationReturnType(LegendInterface legendInterface, PureModelContextData data) + { + PureModel pureModel = legendInterface.compile(data); + return RelationTypeHelper.convert((org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType) pureModel.getConcreteFunctionDefinition(REPL_RUN_FUNCTION_QUALIFIED_PATH, null)._expressionSequence().getLast()._genericType()._typeArguments().getFirst()._rawType()); + } + + public static ValueSpecification parseQuery(String code, Boolean returnSourceInformation) + { + return PureGrammarParser.newInstance().parseValueSpecification(code, "", 0, 0, returnSourceInformation != null && returnSourceInformation); + } + + public static String getQueryCode(ValueSpecification valueSpecification, Boolean pretty) + { + return valueSpecification.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().withRenderStyle(pretty != null && pretty ? RenderStyle.PRETTY : RenderStyle.STANDARD).build()); + } + + public static CompletionResult getCodeTypeahead(String code, Boolean isPartial, PureModelContextData data) + { + try + { + PureModelContextData newData = PureModelContextData.newBuilder() + .withOrigin(data.getOrigin()) + .withSerializer(data.getSerializer()) + .withElements(ListIterate.select(data.getElements(), el -> !el.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH))) + .build(); + String graphCode = PureGrammarComposer.newInstance(PureGrammarComposerContext.Builder.newInstance().build()).renderPureModelContextData(newData); + String queryCode = code; + if (isPartial != null && isPartial) + { + Function func = (Function) ListIterate.select(data.getElements(), el -> el.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH)).getFirst(); + String existingCode = func.body.get(0).accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build()); + queryCode = existingCode + code; + } + CompletionResult result = new Completer(graphCode, Lists.mutable.with(new RelationalCompleterExtension())).complete(queryCode); + if (result.getEngineException() != null) + { + return new CompletionResult(Lists.mutable.empty()); + } + return result; + } + catch (Exception e) + { + return new CompletionResult(Lists.mutable.empty()); + } + } + + /** + * Replace the magic function in the given graph data by a new function with the body of the specified lambda + */ + public static Pair injectNewFunction(PureModelContextData originalData, Lambda lambda) + { + Function originalFunction = (Function) ListIterate.select(originalData.getElements(), e -> e.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH)).getFirst(); + Function func = new Function(); + func.name = originalFunction.name; + func._package = originalFunction._package; + func.parameters = originalFunction.parameters; + func.returnType = originalFunction.returnType; + func.returnMultiplicity = originalFunction.returnMultiplicity; + func.body = lambda != null ? lambda.body : func.body; // if no lambda is specified, we'll just use the original function + + PureModelContextData data = PureModelContextData.newBuilder() + .withOrigin(originalData.getOrigin()) + .withSerializer(originalData.getSerializer()) + .withElements(ListIterate.select(originalData.getElements(), el -> !el.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH))) + .withElement(func) + .build(); + + return Tuples.pair(data, func); + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/REPLServer.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/REPLServer.java new file mode 100644 index 00000000000..844ba61504e --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/REPLServer.java @@ -0,0 +1,104 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.Filter; +import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpServer; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.factory.Maps; +import org.finos.legend.engine.plan.execution.PlanExecutor; +import org.finos.legend.engine.repl.client.Client; +import org.finos.legend.engine.repl.core.commands.Execute; +import org.finos.legend.engine.repl.relational.server.handler.DataCubeInfrastructure; +import org.finos.legend.engine.repl.relational.server.handler.DataCubeQueryBuilder; +import org.finos.legend.engine.repl.relational.server.handler.DataCubeQueryExecutor; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; + +import java.net.InetSocketAddress; +import java.util.List; + +import static org.finos.legend.engine.repl.relational.server.REPLServerHelpers.DEV__CORSFilter; +import static org.jline.jansi.Ansi.ansi; + +public class REPLServer +{ + private static final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); + private static final PlanExecutor planExecutor = PlanExecutor.newPlanExecutorBuilder().withAvailableStoreExecutors().build(); + private final Client client; + private final REPLServerHelpers.REPLServerState state; + + private int port; + private String webAppDevBaseUrl; + + public REPLServer(Client client) + { + this.client = client; + this.state = new REPLServerHelpers.REPLServerState(objectMapper, planExecutor, client.getLegendInterface()); + this.state.setClient(client); + } + + public void setExecuteResult(Execute.ExecuteResult executeResult) + { + this.state.initializeWithREPLExecutedQuery(executeResult); + } + + public String getUrl() + { + String dynamicBaseUrl = System.getenv("LEGEND_REPL_DATA_CUBE_BASE_URL"); + String baseUrl = dynamicBaseUrl != null ? dynamicBaseUrl : this.webAppDevBaseUrl; + return (baseUrl != null ? baseUrl : "http://localhost:" + this.port) + "/repl/dataCube"; + } + + public void initialize() throws Exception + { + InetSocketAddress serverPortAddress = new InetSocketAddress(System.getProperty("legend.repl.dataCube.devPort") != null ? Integer.parseInt(System.getProperty("legend.repl.dataCube.devPort")) : 0); + HttpServer server = HttpServer.create(serverPortAddress, 0); + + // register handlers + MutableList contexts = Maps.mutable.empty() + .withKeyValue("/repl/", new DataCubeInfrastructure.StaticContent()) + .withKeyValue("/api/dataCube/gridLicenseKey", new DataCubeInfrastructure.GridLicenseKey()) + .withKeyValue("/api/dataCube/typeahead", new DataCubeQueryBuilder.QueryTypeahead()) + .withKeyValue("/api/dataCube/parseQuery", new DataCubeQueryBuilder.ParseQuery()) + .withKeyValue("/api/dataCube/getQueryCode", new DataCubeQueryBuilder.GetQueryCode()) + .withKeyValue("/api/dataCube/getQueryCode/batch", new DataCubeQueryBuilder.GetQueryCodeBatch()) + .withKeyValue("/api/dataCube/getBaseQuery", new DataCubeQueryBuilder.GetBaseQuery()) + .withKeyValue("/api/dataCube/getRelationReturnType", new DataCubeQueryBuilder.GetRelationReturnType()) + .withKeyValue("/api/dataCube/executeQuery", new DataCubeQueryExecutor.ExecuteQuery()) + .keyValuesView().collect(config -> server.createContext(config.getOne(), config.getTwo().getHandler(this.state))).toList(); + + // CORS filter + // only needed if we're not serving the webapp from the same server, e.g. in development + if (System.getProperty("legend.repl.dataCube.devWebAppBaseUrl") != null) + { + this.webAppDevBaseUrl = System.getProperty("legend.repl.dataCube.devWebAppBaseUrl"); + List filters = Lists.mutable.empty(); + filters.add(new DEV__CORSFilter(Lists.mutable.with(this.webAppDevBaseUrl))); + contexts.forEach(context -> context.getFilters().addAll(filters)); + } + + server.setExecutor(null); + server.start(); + this.port = server.getAddress().getPort(); + if (this.webAppDevBaseUrl != null) + { + this.client.getTerminal().writer().println(ansi().fgBrightBlack().a("[DEV] DataCube expects webapp at: " + webAppDevBaseUrl).reset()); + } + this.client.getTerminal().writer().println(ansi().fgBrightBlack().a("[DEV] DataCube has started at port: " + this.port).reset()); + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/REPLServerHelpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/REPLServerHelpers.java new file mode 100644 index 00000000000..96f91bcdb63 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/REPLServerHelpers.java @@ -0,0 +1,278 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.Filter; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.pure.grammar.to.DEPRECATED_PureGrammarComposerCore; +import org.finos.legend.engine.plan.execution.PlanExecutor; +import org.finos.legend.engine.plan.execution.result.builder.tds.TDSBuilder; +import org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.SQLExecutionNode; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.application.AppliedFunction; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.ClassInstance; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.PackageableElementPtr; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.relation.ColSpec; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.relation.ColSpecArray; +import org.finos.legend.engine.repl.client.Client; +import org.finos.legend.engine.repl.core.commands.Execute; +import org.finos.legend.engine.repl.core.legend.LegendInterface; +import org.finos.legend.engine.repl.relational.server.model.DataCubeQuery; +import org.finos.legend.engine.repl.relational.server.model.DataCubeQueryColumn; +import org.finos.legend.engine.repl.relational.server.model.DataCubeQuerySourceREPLExecutedQuery; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_QUALIFIED_PATH; + +public class REPLServerHelpers +{ + public static void handleResponse(HttpExchange exchange, int responseCode, String response, REPLServerState state) + { + try + { + OutputStream os = exchange.getResponseBody(); + byte[] byteResponse = response != null ? response.getBytes(StandardCharsets.UTF_8) : new byte[0]; + exchange.sendResponseHeaders(responseCode, byteResponse.length); + os.write(byteResponse); + os.close(); + } + catch (IOException e) + { + state.log(e.getMessage()); + } + } + + public static Map getQueryParams(HttpExchange exchange) + { + String query = exchange.getRequestURI().getQuery(); + Map result = new HashMap<>(); + if (query == null) + { + return result; + } + for (String param : query.split("&")) + { + String[] entry = param.split("="); + if (entry.length > 1) + { + result.put(entry[0], entry[1]); + } + else + { + result.put(entry[0], ""); + } + } + return result; + } + + public static class REPLServerState + { + public final ObjectMapper objectMapper; + public final PlanExecutor planExecutor; + public final LegendInterface legendInterface; + public Long startTime; + + private Client client; + private PureModelContextData currentPureModelContextData; + private DataCubeQuery query; + + public REPLServerState(ObjectMapper objectMapper, PlanExecutor planExecutor, LegendInterface legendInterface) + { + this.objectMapper = objectMapper; + this.planExecutor = planExecutor; + this.legendInterface = legendInterface; + } + + public void initializeWithREPLExecutedQuery(Execute.ExecuteResult executeResult) + { + this.currentPureModelContextData = executeResult.pureModelContextData; + + this.startTime = System.currentTimeMillis(); + this.query = new DataCubeQuery(); + this.query.name = "New Report"; + this.query.configuration = null; // initially, the config is not initialized + + // process source + DataCubeQuerySourceREPLExecutedQuery source = new DataCubeQuerySourceREPLExecutedQuery(); + + if (!(executeResult.result instanceof RelationalResult) || !(((RelationalResult) executeResult.result).builder instanceof TDSBuilder)) + { + throw new RuntimeException("Can't initialize DataCube. Last executed query did not produce a TDS (i.e. data-grid), try a different query..."); + } + + RelationType relationType; + try + { + relationType = (RelationType) executeResult.pureModel.getConcreteFunctionDefinition(REPL_RUN_FUNCTION_QUALIFIED_PATH, null)._expressionSequence().getLast()._genericType()._typeArguments().getFirst()._rawType(); + } + catch (Exception e) + { + throw new RuntimeException("Can't initialize DataCube. Last executed query must return a relation type, try a different query..."); + } + + RelationalResult result = (RelationalResult) executeResult.result; + source.columns = ListIterate.collect(((TDSBuilder) result.builder).columns, col -> new DataCubeQueryColumn(col.name, col.type)); + + // try to extract the runtime for the query + // remove any usage of multiple from(), only add one to the end + // TODO: we might need to account for other variants of ->from(), such as when mapping is specified + Function function = (Function) ListIterate.select(executeResult.pureModelContextData.getElements(), e -> e.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH)).getFirst(); + String runtime = null; + MutableList fns = Lists.mutable.empty(); + ValueSpecification currentExpression = function.body.get(0); + while (currentExpression instanceof AppliedFunction) + { + AppliedFunction fn = (AppliedFunction) currentExpression; + if (fn.function.equals("from")) + { + String newRuntime = ((PackageableElementPtr) fn.parameters.get(1)).fullPath; + if (runtime != null && !runtime.equals(newRuntime)) + { + throw new RuntimeException("Can't initialize DataCube. Source query contains multiple different ->from(), only one is expected"); + } + else + { + runtime = newRuntime; + } + } + else + { + fns.add(fn); + } + currentExpression = fn.parameters.get(0); + } + for (AppliedFunction fn : fns) + { + fn.parameters.set(0, currentExpression); + currentExpression = fn; + } + + this.query.partialQuery = ""; + source.query = currentExpression.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build()); + source.runtime = runtime; + this.query.source = source; + + // build the partial query + // NOTE: for this, the initial query is going to be a select all + AppliedFunction partialFn = new AppliedFunction(); + partialFn.function = "select"; + ColSpecArray colSpecArray = new ColSpecArray(); + colSpecArray.colSpecs = ListIterate.collect(source.columns, col -> + { + ColSpec colSpec = new ColSpec(); + colSpec.name = col.name; + return colSpec; + }); + partialFn.parameters = Lists.mutable.with(new ClassInstance("colSpecArray", colSpecArray, null)); + this.query.partialQuery = partialFn.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build()); + + // build the full query + AppliedFunction fullFn = new AppliedFunction(); + fullFn.function = "from"; + fullFn.parameters = Lists.mutable.with(partialFn, new PackageableElementPtr(runtime)); + partialFn.parameters = Lists.mutable.with(currentExpression).withAll(partialFn.parameters); + this.query.query = fullFn.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build()); + } + + public PureModelContextData getCurrentPureModelContextData() + { + PureModelContextData data = this.currentPureModelContextData; + if (data == null) + { + throw new RuntimeException("Can't retrieve current graph data. Try to load or run a query in REPL before launching DataCube..."); + } + return data; + } + + public DataCubeQuery getQuery() + { + return this.query; + } + + public void setClient(Client client) + { + this.client = client; + } + + public void log(String message) + { + if (this.client != null) + { + this.client.getTerminal().writer().println(message); + } + } + } + + public interface DataCubeServerHandler + { + HttpHandler getHandler(REPLServerState state); + } + + public static class DEV__CORSFilter extends Filter + { + private final MutableList allowedOrigins; + + DEV__CORSFilter(MutableList allowedOrigins) + { + super(); + if (allowedOrigins.isEmpty()) + { + throw new IllegalArgumentException("Can't configure CORS filter: Allowed origins cannot be empty"); + } + this.allowedOrigins = allowedOrigins; + } + + @Override + public void doFilter(HttpExchange exchange, Chain chain) throws IOException + { + Headers headers = exchange.getResponseHeaders(); + headers.add("Access-Control-Allow-Origin", allowedOrigins.makeString(",")); + headers.add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS"); + headers.add("Access-Control-Allow-Credentials", "true"); + headers.add("chainPreflight", "false"); + headers.add("Access-Control-Allow-Headers", "Access-Control-Allow-Headers, Origin, Accept, X-Requested-With, Content-Type, Access-Control-Request-Method, Access-Control-Request-Headers"); + + if (exchange.getRequestMethod().equalsIgnoreCase("OPTIONS")) + { + exchange.sendResponseHeaders(204, -1); + } + else + { + chain.doFilter(exchange); + } + } + + @Override + public String description() + { + return "CORSFilter"; + } + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeInfrastructure.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeInfrastructure.java new file mode 100644 index 00000000000..b46c1a86736 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeInfrastructure.java @@ -0,0 +1,98 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.handler; + +import com.sun.net.httpserver.HttpHandler; +import org.apache.commons.io.IOUtils; +import org.finos.legend.engine.repl.relational.server.REPLServer; + +import java.io.InputStream; +import java.io.OutputStream; + +import static org.finos.legend.engine.repl.relational.server.REPLServerHelpers.*; + +public class DataCubeInfrastructure +{ + public static class GridLicenseKey implements DataCubeServerHandler + { + @Override + public HttpHandler getHandler(REPLServerState state) + { + return exchange -> + { + if ("GET".equals(exchange.getRequestMethod())) + { + try + { + String licenseKey = System.getProperty("legend.repl.dataCube.gridLicenseKey") == null ? "" : System.getProperty("legend.repl.dataCube.gridLicenseKey"); + String key = state.objectMapper.writeValueAsString(licenseKey); + handleResponse(exchange, 200, key, state); + } + catch (Exception e) + { + handleResponse(exchange, 500, e.getMessage(), state); + } + } + }; + } + } + + public static class StaticContent implements DataCubeServerHandler + { + @Override + public HttpHandler getHandler(REPLServerState state) + { + return exchange -> + { + if ("GET".equals(exchange.getRequestMethod())) + { + String[] path = exchange.getRequestURI().getPath().split("/repl/"); + String resourcePath = "/web-content/dist/repl/" + (path[1].equals("dataCube") ? "index.html" : path[1]); + try (OutputStream os = exchange.getResponseBody(); + InputStream is = REPLServer.class.getResourceAsStream(resourcePath) + ) + { + if (is == null) + { + exchange.sendResponseHeaders(404, -1); + } + else + { + if (resourcePath.endsWith(".html")) + { + exchange.getResponseHeaders().add("Content-Type", "text/html; charset=utf-8"); + } + if (resourcePath.endsWith(".js")) + { + exchange.getResponseHeaders().add("Content-Type", "text/javascript; charset=utf-8"); + } + else if (resourcePath.endsWith(".css")) + { + exchange.getResponseHeaders().add("Content-Type", "text/css; charset=utf-8"); + } + + exchange.sendResponseHeaders(200, 0); + IOUtils.copy(is, os); + } + } + catch (Exception e) + { + handleResponse(exchange, 500, e.getMessage(), state); + } + } + }; + } + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryBuilder.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryBuilder.java new file mode 100644 index 00000000000..21250f141cb --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryBuilder.java @@ -0,0 +1,217 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.handler; + +import com.sun.net.httpserver.HttpHandler; +import org.eclipse.collections.impl.map.mutable.MapAdapter; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; +import org.finos.legend.engine.repl.autocomplete.CompletionResult; +import org.finos.legend.engine.repl.relational.server.DataCubeHelpers; +import org.finos.legend.engine.repl.relational.server.model.*; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +import static org.finos.legend.engine.repl.relational.server.REPLServerHelpers.*; + +public class DataCubeQueryBuilder +{ + public static class ParseQuery implements DataCubeServerHandler + { + @Override + public HttpHandler getHandler(REPLServerState state) + { + return exchange -> + { + if ("POST".equals(exchange.getRequestMethod())) + { + try + { + InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); + BufferedReader bufferReader = new BufferedReader(inputStreamReader); + String requestBody = bufferReader.lines().collect(Collectors.joining()); + DataCubeParseQueryInput input = state.objectMapper.readValue(requestBody, DataCubeParseQueryInput.class); + ValueSpecification result = DataCubeHelpers.parseQuery(input.code, input.returnSourceInformation); + handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state); + } + catch (Exception e) + { + handleResponse(exchange, 400, e instanceof EngineException ? state.objectMapper.writeValueAsString(e) : e.getMessage(), state); + } + } + }; + } + } + + public static class GetQueryCode implements DataCubeServerHandler + { + @Override + public HttpHandler getHandler(REPLServerState state) + { + return exchange -> + { + if ("POST".equals(exchange.getRequestMethod())) + { + try + { + InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); + BufferedReader bufferReader = new BufferedReader(inputStreamReader); + String requestBody = bufferReader.lines().collect(Collectors.joining()); + DataCubeGetQueryCodeInput input = state.objectMapper.readValue(requestBody, DataCubeGetQueryCodeInput.class); + handleResponse(exchange, 200, DataCubeHelpers.getQueryCode(input.query, input.pretty), state); + } + catch (Exception e) + { + handleResponse(exchange, 400, e.getMessage(), state); + } + } + }; + } + } + + public static class GetQueryCodeBatch implements DataCubeServerHandler + { + @Override + public HttpHandler getHandler(REPLServerState state) + { + return exchange -> + { + if ("POST".equals(exchange.getRequestMethod())) + { + try + { + InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); + BufferedReader bufferReader = new BufferedReader(inputStreamReader); + String requestBody = bufferReader.lines().collect(Collectors.joining()); + DataCubeGetQueryCodeBatchInput input = state.objectMapper.readValue(requestBody, DataCubeGetQueryCodeBatchInput.class); + DataCubeGetQueryCodeBatchResult result = new DataCubeGetQueryCodeBatchResult(); + MapAdapter.adapt(input.queries).forEachKeyValue((key, value) -> + { + try + { + result.queries.put(key, DataCubeHelpers.getQueryCode(value, input.pretty)); + } + catch (Exception e) + { + result.queries.put(key, null); + } + }); + handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state); + } + catch (Exception e) + { + handleResponse(exchange, 400, e.getMessage(), state); + } + } + }; + } + } + + public static class QueryTypeahead implements DataCubeServerHandler + { + @Override + public HttpHandler getHandler(REPLServerState state) + { + return exchange -> + { + if ("POST".equals(exchange.getRequestMethod())) + { + try + { + InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); + BufferedReader bufferReader = new BufferedReader(inputStreamReader); + String requestBody = bufferReader.lines().collect(Collectors.joining()); + DataCubeQueryTypeaheadInput input = state.objectMapper.readValue(requestBody, DataCubeQueryTypeaheadInput.class); + PureModelContextData data = state.getCurrentPureModelContextData(); + CompletionResult result = DataCubeHelpers.getCodeTypeahead(input.code, input.isPartial, data); + handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result.getCompletion()), state); + } + catch (Exception e) + { + handleResponse(exchange, 500, e.getMessage(), state); + } + } + }; + } + } + + public static class GetRelationReturnType implements DataCubeServerHandler + { + @Override + public HttpHandler getHandler(REPLServerState state) + { + return exchange -> + { + if ("POST".equals(exchange.getRequestMethod())) + { + try + { + InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); + BufferedReader bufferReader = new BufferedReader(inputStreamReader); + String requestBody = bufferReader.lines().collect(Collectors.joining()); + DataCubeGetQueryRelationReturnTypeInput input = state.objectMapper.readValue(requestBody, DataCubeGetQueryRelationReturnTypeInput.class); + Lambda lambda = input.query; // if no lambda is specified, we're executing the initial query + PureModelContextData data = DataCubeHelpers.injectNewFunction(state.getCurrentPureModelContextData(), lambda).getOne(); + handleResponse(exchange, 200, state.objectMapper.writeValueAsString(DataCubeHelpers.getRelationReturnType(state.legendInterface, data)), state); + } + catch (Exception e) + { + handleResponse(exchange, 500, e instanceof EngineException ? state.objectMapper.writeValueAsString(e) : e.getMessage(), state); + } + } + }; + } + } + + public static class GetBaseQuery implements DataCubeServerHandler + { + @Override + public HttpHandler getHandler(REPLServerState state) + { + return exchange -> + { + if ("GET".equals(exchange.getRequestMethod())) + { + try + { + DataCubeQuery query = state.getQuery(); + if (query != null) + { + DataCubeGetBaseQueryResult result = new DataCubeGetBaseQueryResult(); + result.timestamp = state.startTime; + result.query = query; + result.partialQuery = DataCubeHelpers.parseQuery(query.partialQuery, false); + result.sourceQuery = DataCubeHelpers.parseQuery(query.source.query, false); + handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state); + } + else + { + throw new RuntimeException("DataCube base query has not been set!"); + } + } + catch (Exception e) + { + handleResponse(exchange, 500, e.getMessage(), state); + } + } + }; + } + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryExecutor.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryExecutor.java new file mode 100644 index 00000000000..bcd5a03cc6c --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/handler/DataCubeQueryExecutor.java @@ -0,0 +1,62 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.handler; + +import com.sun.net.httpserver.HttpHandler; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; +import org.finos.legend.engine.repl.relational.server.DataCubeHelpers; +import org.finos.legend.engine.repl.relational.server.model.DataCubeExecutionInput; +import org.finos.legend.engine.repl.relational.server.model.DataCubeExecutionResult; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +import static org.finos.legend.engine.repl.relational.server.DataCubeHelpers.executeQuery; +import static org.finos.legend.engine.repl.relational.server.REPLServerHelpers.*; + +public class DataCubeQueryExecutor +{ + public static class ExecuteQuery implements DataCubeServerHandler + { + @Override + public HttpHandler getHandler(REPLServerState state) + { + return exchange -> + { + if ("POST".equals(exchange.getRequestMethod())) + { + try + { + InputStreamReader inputStreamReader = new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8); + BufferedReader bufferReader = new BufferedReader(inputStreamReader); + String requestBody = bufferReader.lines().collect(Collectors.joining()); + DataCubeExecutionInput input = state.objectMapper.readValue(requestBody, DataCubeExecutionInput.class); + Lambda lambda = input.query; + PureModelContextData data = DataCubeHelpers.injectNewFunction(state.getCurrentPureModelContextData(), lambda).getOne(); + DataCubeExecutionResult result = executeQuery(state.legendInterface, state.planExecutor, data); + handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state); + } + catch (Exception e) + { + handleResponse(exchange, 500, e.getMessage(), state); + } + } + }; + } + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeExecutionInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeExecutionInput.java new file mode 100644 index 00000000000..2df93d37859 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeExecutionInput.java @@ -0,0 +1,22 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; + +public class DataCubeExecutionInput +{ + public Lambda query; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeExecutionResult.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeExecutionResult.java new file mode 100644 index 00000000000..857f9ed0dc2 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeExecutionResult.java @@ -0,0 +1,20 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +public class DataCubeExecutionResult +{ + public String result; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetBaseQueryResult.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetBaseQueryResult.java new file mode 100644 index 00000000000..21ab4acbdde --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetBaseQueryResult.java @@ -0,0 +1,25 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; + +public class DataCubeGetBaseQueryResult +{ + public DataCubeQuery query; + public Long timestamp; + public ValueSpecification partialQuery; + public ValueSpecification sourceQuery; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryCodeBatchInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryCodeBatchInput.java new file mode 100644 index 00000000000..04825178200 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryCodeBatchInput.java @@ -0,0 +1,25 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; + +import java.util.Map; + +public class DataCubeGetQueryCodeBatchInput +{ + public Map queries; + public Boolean pretty; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryCodeBatchResult.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryCodeBatchResult.java new file mode 100644 index 00000000000..2b3a8879284 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryCodeBatchResult.java @@ -0,0 +1,22 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +import java.util.Map; + +public class DataCubeGetQueryCodeBatchResult +{ + public Map queries; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryCodeInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryCodeInput.java new file mode 100644 index 00000000000..0f43bcc4d41 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryCodeInput.java @@ -0,0 +1,23 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; + +public class DataCubeGetQueryCodeInput +{ + public ValueSpecification query; + public Boolean pretty; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryRelationReturnTypeInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryRelationReturnTypeInput.java new file mode 100644 index 00000000000..e44383e2cd3 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeGetQueryRelationReturnTypeInput.java @@ -0,0 +1,22 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; + +public class DataCubeGetQueryRelationReturnTypeInput +{ + public Lambda query; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeParseQueryInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeParseQueryInput.java new file mode 100644 index 00000000000..5fad8067b2a --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeParseQueryInput.java @@ -0,0 +1,21 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +public class DataCubeParseQueryInput +{ + public String code; + public Boolean returnSourceInformation; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQuery.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQuery.java new file mode 100644 index 00000000000..da9635b6507 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQuery.java @@ -0,0 +1,31 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +import com.fasterxml.jackson.annotation.JsonRawValue; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +public class DataCubeQuery +{ + public String name; + public String query; + public String partialQuery; + public DataCubeQuerySource source; + + // NOTE: we don't need to process the config, so we will leave it as raw JSON + @JsonRawValue + @JsonDeserialize(using = DataCubeQueryConfigurationDeserializer.class) + public String configuration; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQueryColumn.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQueryColumn.java new file mode 100644 index 00000000000..7b7bf40d0e2 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQueryColumn.java @@ -0,0 +1,27 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +public class DataCubeQueryColumn +{ + public String name; + public String type; + + public DataCubeQueryColumn(String name, String type) + { + this.name = name; + this.type = type; + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQueryConfigurationDeserializer.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQueryConfigurationDeserializer.java new file mode 100644 index 00000000000..812eb87b4b5 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQueryConfigurationDeserializer.java @@ -0,0 +1,32 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.TreeNode; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; + +import java.io.IOException; + +public class DataCubeQueryConfigurationDeserializer extends JsonDeserializer +{ + @Override + public String deserialize(JsonParser jp, DeserializationContext ctx) throws IOException + { + TreeNode tree = jp.getCodec().readTree(jp); + return tree.toString(); + } +} \ No newline at end of file diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQuerySource.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQuerySource.java new file mode 100644 index 00000000000..d6239b9e754 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQuerySource.java @@ -0,0 +1,31 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.util.List; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = DataCubeQuerySourceREPLExecutedQuery.class, name = "REPLExecutedQuery") +}) +public abstract class DataCubeQuerySource +{ + public String query; + public String runtime; + public List columns; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQuerySourceREPLExecutedQuery.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQuerySourceREPLExecutedQuery.java new file mode 100644 index 00000000000..c958c05bc89 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQuerySourceREPLExecutedQuery.java @@ -0,0 +1,19 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +public class DataCubeQuerySourceREPLExecutedQuery extends DataCubeQuerySource +{ +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQueryTypeaheadInput.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQueryTypeaheadInput.java new file mode 100644 index 00000000000..ca3afec59e5 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/server/model/DataCubeQueryTypeaheadInput.java @@ -0,0 +1,21 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.relational.server.model; + +public class DataCubeQueryTypeaheadInput +{ + public String code; + public Boolean isPartial; +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java index a6d4c822ffb..172a4d63162 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java @@ -47,21 +47,21 @@ public void testAutocompleteFunctionParameter() @Test public void testArrowOnFunction() { - Assert.assertEquals("[distinct , distinct], [drop , drop], [select , select], [extend , extend], [filter , filter], [from , from], [groupBy , groupBy], [join , join], [limit , limit], [rename , rename], [size , size], [slice , slice], [sort , sort]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->filter(x|$x.col == 'oo')->"))); + Assert.assertEquals("[distinct , distinct(], [drop , drop(], [select , select(], [extend , extend(], [filter , filter(], [from , from(], [groupBy , groupBy(], [join , join(], [limit , limit(], [rename , rename(], [size , size(], [slice , slice(], [sort , sort(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->filter(x|$x.col == 'oo')->"))); Assert.assertEquals("PARSER error at [6:1-23]: parsing error", new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->limit(10)-").getEngineException().toPretty()); } @Test public void testArrowRelation() { - Assert.assertEquals("[distinct , distinct], [drop , drop], [select , select], [extend , extend], [filter , filter], [from , from], [groupBy , groupBy], [join , join], [limit , limit], [rename , rename], [size , size], [slice , slice], [sort , sort]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->"))); - Assert.assertEquals("[select , select], [size , size], [slice , slice], [sort , sort]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->s"))); + Assert.assertEquals("[distinct , distinct(], [drop , drop(], [select , select(], [extend , extend(], [filter , filter(], [from , from(], [groupBy , groupBy(], [join , join(], [limit , limit(], [rename , rename(], [size , size(], [slice , slice(], [sort , sort(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->"))); + Assert.assertEquals("[select , select(], [size , size(], [slice , slice(], [sort , sort(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->s"))); } @Test public void testArrowDeep() { - Assert.assertEquals("[contains , contains], [startsWith , startsWith], [endsWith , endsWith], [toLower , toLower], [toUpper , toUpper], [lpad , lpad], [rpad , rpad], [parseInteger , parseInteger], [parseFloat , parseFloat]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->filter(f|$f.col->"))); + Assert.assertEquals("[contains , contains(], [startsWith , startsWith(], [endsWith , endsWith(], [toLower , toLower(], [toUpper , toUpper(], [lpad , lpad(], [rpad , rpad(], [parseInteger , parseInteger(], [parseFloat , parseFloat(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->filter(f|$f.col->"))); } @Test @@ -82,13 +82,13 @@ public void testDeepWithCompilationError() @Test public void testArrowPostCol() { - Assert.assertEquals("[ascending , ascending], [descending , descending]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->sort(~col->"))); + Assert.assertEquals("[ascending , ascending(], [descending , descending(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->sort(~col->"))); } @Test public void testMultiLevelFunction() { - Assert.assertEquals("[select , select]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->select(~[col])->join(#>{a::A.t}#->selec"))); + Assert.assertEquals("[select , select(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->select(~[col])->join(#>{a::A.t}#->selec"))); Assert.assertEquals("[col , col]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->select(~[col])->join(#>{a::A.t}#->select(~"))); } @@ -104,7 +104,6 @@ public void testDotInFilterDeepRelation() Assert.assertEquals("['na col' , 'na col']", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(\"na col\" VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->filter(x|$x.'na"))); } - //-------- // Rename //-------- @@ -141,10 +140,9 @@ public void testGroupBy() Assert.assertEquals("[col , col], [val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->groupBy(~[col,"))); Assert.assertEquals("[col , col], [val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->groupBy(~col, ~z:x|$x."))); Assert.assertEquals("[val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->groupBy(~col, ~[z:x|$x.v"))); - Assert.assertEquals("[sum , sum], [mean , mean], [average , average], [min , min], [max , max], [count , count], [percentile , percentile], [variancePopulation , variancePopulation], [varianceSample , varianceSample], [stdDevPopulation , stdDevPopulation], [stdDevSample , stdDevSample]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->groupBy(~col, ~[z:x|$x.val:y|$y->"))); + Assert.assertEquals("[sum , sum(], [mean , mean(], [average , average(], [min , min(], [max , max(], [count , count(], [percentile , percentile(], [variancePopulation , variancePopulation(], [varianceSample , varianceSample(], [stdDevPopulation , stdDevPopulation(], [stdDevSample , stdDevSample(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->groupBy(~col, ~[z:x|$x.val:y|$y->"))); } - //------ // Join //------ @@ -169,8 +167,8 @@ public void testSelect() { Assert.assertEquals("[col , col]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->select(~c"))); Assert.assertEquals("[col , col], [val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->select(~[col,"))); - Assert.assertEquals("[from , from]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(\"col space\" VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->select(~'col space')->fro"))); - Assert.assertEquals("[from , from]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(\"col space\" VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->select(~['col space'])->fro"))); + Assert.assertEquals("[from , from(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(\"col space\" VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->select(~'col space')->fro"))); + Assert.assertEquals("[from , from(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(\"col space\" VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->select(~['col space'])->fro"))); } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestDataCubeHelpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestDataCubeHelpers.java new file mode 100644 index 00000000000..39b51301e05 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestDataCubeHelpers.java @@ -0,0 +1,287 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.finos.legend.engine.plan.execution.PlanExecutor; +import org.finos.legend.engine.plan.execution.stores.relational.serialization.RelationalResultToJsonDefaultSerializer; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; +import org.finos.legend.engine.repl.core.legend.LegendInterface; +import org.finos.legend.engine.repl.core.legend.LocalLegendInterface; +import org.finos.legend.engine.repl.relational.server.DataCubeHelpers; +import org.finos.legend.engine.repl.relational.server.model.DataCubeExecutionResult; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_SIGNATURE; +import static org.finos.legend.engine.repl.relational.server.DataCubeHelpers.executeQuery; + +public class TestDataCubeHelpers +{ + private static final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); + private static final PlanExecutor planExecutor = PlanExecutor.newPlanExecutorBuilder().withAvailableStoreExecutors().build(); + private final LegendInterface legendInterface = new LocalLegendInterface(); + private final PureModelContextData pureModelContextData = legendInterface.parse( + "###Relational\n" + + "Database test::TestDatabase\n" + + "(\n" + + " Table TEST0\n" + + " (\n" + + " FIRSTNAME VARCHAR(200),\n" + + " LASTNAME VARCHAR(200)\n" + + " )\n" + + ")\n" + + "\n" + + "###Pure\n" + + "function " + REPL_RUN_FUNCTION_SIGNATURE + "\n" + + "{\n" + + " #>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->select(~FIRSTNAME)->from(test::test)\n" + + "}\n" + + "\n" + + "###Runtime\n" + + "Runtime test::test\n" + + "{\n" + + " mappings: [];\n" + + " connections:\n" + + " [\n" + + " test::TestDatabase:\n" + + " [ \n" + + " connection: test::TestConnection\n" + + " ]\n" + + "\n" + + " ];\n" + + "}\n" + + "\n" + + "\n" + + "###Connection\n" + + "RelationalDatabaseConnection test::TestConnection\n" + + "{\n" + + " store: test::TestDatabase;\n" + + " type: H2;\n" + + " specification: LocalH2\n" + + " {\n" + + " testDataSetupSqls: [\n" + + " '\\nDrop table if exists TEST0;\\nCreate Table TEST0(FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200));\\nInsert into TEST0 (FIRSTNAME, LASTNAME) values (\\'John\\', \\'Doe\\');\\nInsert into TEST0 (FIRSTNAME, LASTNAME) values (\\'Tim\\', \\'Smith\\');\\nInsert into TEST0 (FIRSTNAME, LASTNAME) values (\\'Nicole\\', \\'Doe\\');\\n\\n'\n" + + " ];\n" + + " };\n" + + " auth: DefaultH2;\n" + + "}" + ); + + @Test + public void testExecuteSort() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(test::test)->sort([~FIRSTNAME->ascending()])"; + String expectedResult = "{\"builder\":{\"_type\":\"tdsBuilder\",\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"}]},\"activities\":[{\"_type\":\"relational\",\"sql\":\"select \\\"test0_0\\\".FIRSTNAME as \\\"FIRSTNAME\\\", \\\"test0_0\\\".LASTNAME as \\\"LASTNAME\\\" from TEST0 as \\\"test0_0\\\" where (\\\"test0_0\\\".FIRSTNAME <> 'Doe' OR \\\"test0_0\\\".FIRSTNAME is null) order by \\\"FIRSTNAME\\\"\"}],\"result\":{\"columns\":[\"FIRSTNAME\",\"LASTNAME\"],\"rows\":[{\"values\":[\"John\",\"Doe\"]},{\"values\":[\"Nicole\",\"Doe\"]},{\"values\":[\"Tim\",\"Smith\"]}]}}"; + testExecuteQuery(expectedResult, lambda); + } + + @Test + public void testExecuteFilter() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | ($c.FIRSTNAME != 'Doe' && $c.LASTNAME != 'Doe'))->from(test::test)->sort([~FIRSTNAME->ascending()])"; + String expectedResult = "{\"builder\":{\"_type\":\"tdsBuilder\",\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"}]},\"activities\":[{\"_type\":\"relational\",\"sql\":\"select \\\"test0_0\\\".FIRSTNAME as \\\"FIRSTNAME\\\", \\\"test0_0\\\".LASTNAME as \\\"LASTNAME\\\" from TEST0 as \\\"test0_0\\\" where ((\\\"test0_0\\\".FIRSTNAME <> 'Doe' OR \\\"test0_0\\\".FIRSTNAME is null) and (\\\"test0_0\\\".LASTNAME <> 'Doe' OR \\\"test0_0\\\".LASTNAME is null)) order by \\\"FIRSTNAME\\\"\"}],\"result\":{\"columns\":[\"FIRSTNAME\",\"LASTNAME\"],\"rows\":[{\"values\":[\"Tim\",\"Smith\"]}]}}"; + testExecuteQuery(expectedResult, lambda); + } + + @Test + public void testExecuteGroupBy() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(test::test)->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()])"; + String expectedResult = "{\"builder\":{\"_type\":\"tdsBuilder\",\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"count\",\"type\":\"Integer\"}]},\"activities\":[{\"_type\":\"relational\",\"sql\":\"select \\\"test0_0\\\".FIRSTNAME as \\\"FIRSTNAME\\\", count(\\\"test0_0\\\".FIRSTNAME) as \\\"count\\\" from TEST0 as \\\"test0_0\\\" where (\\\"test0_0\\\".FIRSTNAME <> 'Doe' OR \\\"test0_0\\\".FIRSTNAME is null) group by \\\"FIRSTNAME\\\"\"}],\"result\":{\"columns\":[\"FIRSTNAME\",\"count\"],\"rows\":[{\"values\":[\"John\",1]},{\"values\":[\"Nicole\",1]},{\"values\":[\"Tim\",1]}]}}"; + testExecuteQuery(expectedResult, lambda); + } + + private void testExecuteQuery(String expectedResult, String code) + { + try + { + Lambda lambda = (Lambda) DataCubeHelpers.parseQuery(code, false); + PureModelContextData data = DataCubeHelpers.injectNewFunction(pureModelContextData, lambda).getOne(); + DataCubeExecutionResult result = executeQuery(legendInterface, planExecutor, data); + Assert.assertEquals(expectedResult, RelationalResultToJsonDefaultSerializer.removeComment(result.result)); + } + catch (IOException e) + { + throw new RuntimeException(e.getMessage()); + } + } + + @Test + public void testParseQuerySimple() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(test::test)->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()])"; + String expectedQuery = "{\"_type\":\"lambda\",\"body\":[{\"_type\":\"func\",\"function\":\"groupBy\",\"parameters\":[{\"_type\":\"func\",\"function\":\"from\",\"parameters\":[{\"_type\":\"func\",\"function\":\"filter\",\"parameters\":[{\"_type\":\"classInstance\",\"type\":\">\",\"value\":{\"path\":[\"test::TestDatabase\",\"TEST0\"]}},{\"_type\":\"lambda\",\"body\":[{\"_type\":\"func\",\"function\":\"not\",\"parameters\":[{\"_type\":\"func\",\"function\":\"equal\",\"parameters\":[{\"_type\":\"property\",\"parameters\":[{\"_type\":\"var\",\"name\":\"c\"}],\"property\":\"FIRSTNAME\"},{\"_type\":\"string\",\"value\":\"Doe\"}]}]}],\"parameters\":[{\"_type\":\"var\",\"name\":\"c\"}]}]},{\"_type\":\"packageableElementPtr\",\"fullPath\":\"test::test\"}]},{\"_type\":\"classInstance\",\"type\":\"colSpecArray\",\"value\":{\"colSpecs\":[{\"name\":\"FIRSTNAME\"}]}},{\"_type\":\"classInstance\",\"type\":\"colSpecArray\",\"value\":{\"colSpecs\":[{\"function1\":{\"_type\":\"lambda\",\"body\":[{\"_type\":\"property\",\"parameters\":[{\"_type\":\"var\",\"name\":\"x\"}],\"property\":\"FIRSTNAME\"}],\"parameters\":[{\"_type\":\"var\",\"name\":\"x\"}]},\"function2\":{\"_type\":\"lambda\",\"body\":[{\"_type\":\"func\",\"function\":\"count\",\"parameters\":[{\"_type\":\"var\",\"name\":\"y\"}]}],\"parameters\":[{\"_type\":\"var\",\"name\":\"y\"}]},\"name\":\"count\"}]}}]}],\"parameters\":[]}"; + testParseQuery(expectedQuery, lambda, false); + } + + @Test + public void testParseQuerySimpleWithSourceInformationReturned() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(test::test)->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()])"; + String expectedQuery = "{\"_type\":\"lambda\",\"body\":[{\"_type\":\"func\",\"function\":\"groupBy\",\"parameters\":[{\"_type\":\"func\",\"function\":\"from\",\"parameters\":[{\"_type\":\"func\",\"function\":\"filter\",\"parameters\":[{\"_type\":\"classInstance\",\"sourceInformation\":{\"endColumn\":30,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":2,\"startLine\":1},\"type\":\">\",\"value\":{\"path\":[\"test::TestDatabase\",\"TEST0\"],\"sourceInformation\":{\"endColumn\":30,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":2,\"startLine\":1}}},{\"_type\":\"lambda\",\"body\":[{\"_type\":\"func\",\"function\":\"not\",\"parameters\":[{\"_type\":\"func\",\"function\":\"equal\",\"parameters\":[{\"_type\":\"property\",\"parameters\":[{\"_type\":\"var\",\"name\":\"c\",\"sourceInformation\":{\"endColumn\":45,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":44,\"startLine\":1}}],\"property\":\"FIRSTNAME\",\"sourceInformation\":{\"endColumn\":55,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":47,\"startLine\":1}},{\"_type\":\"string\",\"sourceInformation\":{\"endColumn\":64,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":60,\"startLine\":1},\"value\":\"Doe\"}]}],\"sourceInformation\":{\"endColumn\":58,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":57,\"startLine\":1}}],\"parameters\":[{\"_type\":\"var\",\"name\":\"c\"}],\"sourceInformation\":{\"endColumn\":64,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":42,\"startLine\":1}}],\"sourceInformation\":{\"endColumn\":38,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":33,\"startLine\":1}},{\"_type\":\"packageableElementPtr\",\"fullPath\":\"test::test\",\"sourceInformation\":{\"endColumn\":82,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":73,\"startLine\":1}}],\"sourceInformation\":{\"endColumn\":71,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":68,\"startLine\":1}},{\"_type\":\"classInstance\",\"sourceInformation\":{\"endColumn\":105,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":95,\"startLine\":1},\"type\":\"colSpecArray\",\"value\":{\"colSpecs\":[{\"name\":\"FIRSTNAME\",\"sourceInformation\":{\"endColumn\":104,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":96,\"startLine\":1}}]}},{\"_type\":\"classInstance\",\"sourceInformation\":{\"endColumn\":151,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":109,\"startLine\":1},\"type\":\"colSpecArray\",\"value\":{\"colSpecs\":[{\"function1\":{\"_type\":\"lambda\",\"body\":[{\"_type\":\"property\",\"parameters\":[{\"_type\":\"var\",\"name\":\"x\",\"sourceInformation\":{\"endColumn\":122,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":121,\"startLine\":1}}],\"property\":\"FIRSTNAME\",\"sourceInformation\":{\"endColumn\":132,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":124,\"startLine\":1}}],\"parameters\":[{\"_type\":\"var\",\"name\":\"x\"}],\"sourceInformation\":{\"endColumn\":132,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":119,\"startLine\":1}},\"function2\":{\"_type\":\"lambda\",\"body\":[{\"_type\":\"func\",\"function\":\"count\",\"parameters\":[{\"_type\":\"var\",\"name\":\"y\",\"sourceInformation\":{\"endColumn\":141,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":140,\"startLine\":1}}],\"sourceInformation\":{\"endColumn\":148,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":144,\"startLine\":1}}],\"parameters\":[{\"_type\":\"var\",\"name\":\"y\"}],\"sourceInformation\":{\"endColumn\":150,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":138,\"startLine\":1}},\"name\":\"count\",\"sourceInformation\":{\"endColumn\":150,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":110,\"startLine\":1}}]}}],\"sourceInformation\":{\"endColumn\":92,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":86,\"startLine\":1}}],\"parameters\":[],\"sourceInformation\":{\"endColumn\":153,\"endLine\":1,\"sourceId\":\"\",\"startColumn\":1,\"startLine\":1}}"; + testParseQuery(expectedQuery, lambda, true); + } + + private void testParseQuery(String expectedQuery, String code, boolean returnSourceInformation) + { + try + { + Assert.assertEquals(expectedQuery, objectMapper.writeValueAsString(DataCubeHelpers.parseQuery(code, returnSourceInformation))); + } + catch (IOException e) + { + throw new RuntimeException(e.getMessage()); + } + } + + @Test + public void testGetQueryCodeStandard() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(test::test)->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()])"; + String expectedQuery = "|#>{test::TestDatabase.TEST0}#->filter(c|!($c.FIRSTNAME == 'Doe'))->from(test::test)->groupBy(~[FIRSTNAME], ~[count:x|$x.FIRSTNAME:y|$y->count()])"; + testGetQueryCode(expectedQuery, lambda, false); + } + + @Test + public void testGetQueryCodePretty() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(test::test)->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()])"; + String expectedQuery = "|#>{test::TestDatabase.TEST0}#->filter(\n" + + " c|!($c.FIRSTNAME == 'Doe')\n" + + ")->from(\n" + + " test::test\n" + + ")->groupBy(\n" + + " ~[\n" + + " FIRSTNAME\n" + + " ],\n" + + " ~[\n" + + " count: x|$x.FIRSTNAME:y|$y->count()\n" + + " ]\n" + + ")"; + testGetQueryCode(expectedQuery, lambda, true); + } + + private void testGetQueryCode(String expectedQuery, String code, boolean pretty) + { + ValueSpecification query = DataCubeHelpers.parseQuery(code, false); + Assert.assertEquals(expectedQuery, DataCubeHelpers.getQueryCode(query, pretty)); + } + + @Test + public void testTypeaheadPartial() + { + String code = "->extend(~[newCol:c|'ok', colX: c|$c."; + String expectedResult = "{\"completion\":[{\"completion\":\"FIRSTNAME\",\"display\":\"FIRSTNAME\"}]}"; + testTypeahead(expectedResult, code, true); + } + + @Test + public void testTypeaheadFull() + { + String code = "#>{test::TestDatabase.TEST0}#->extend(~[newCol:c|'ok', colX: c|$c."; + String expectedResult = "{\"completion\":[{\"completion\":\"FIRSTNAME\",\"display\":\"FIRSTNAME\"},{\"completion\":\"LASTNAME\",\"display\":\"LASTNAME\"}]}"; + testTypeahead(expectedResult, code, false); + } + + @Test + public void testTypeaheadFullWithError() + { + String code = "#>{test::TestDatabase.TEST0}#-->extend(~[newCol:c|'ok', colX: c|$c."; + String expectedResult = "{\"completion\":[]}"; + testTypeahead(expectedResult, code, false); + } + + private void testTypeahead(String expectedResult, String code, boolean isPartial) + { + try + { + Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getCodeTypeahead(code, isPartial, pureModelContextData))); + } + catch (IOException e) + { + throw new RuntimeException(e.getMessage()); + } + } + + @Test + public void testExtractRelationReturnTypeGroupBy() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(test::test)->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()])"; + String expectedResult = "{\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"count\",\"type\":\"Integer\"}]}"; + testExtractRelationReturnType(expectedResult, lambda); + } + + @Test + public void testExtractRelationReturnTypeSimpleExtend() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->extend(~newCol:c|'ok')"; + String expectedResult = "{\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"},{\"name\":\"newCol\",\"type\":\"String\"}]}"; + testExtractRelationReturnType(expectedResult, lambda); + } + + @Test + public void testExtractRelationReturnTypeMultipleExtend() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->extend(~[newCol:c|'ok', colX: c|$c.FIRSTNAME])"; + String expectedResult = "{\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"},{\"name\":\"newCol\",\"type\":\"String\"},{\"name\":\"colX\",\"type\":\"String\"}]}"; + testExtractRelationReturnType(expectedResult, lambda); + } + + private void testExtractRelationReturnType(String expectedResult, String code) + { + try + { + Lambda lambda = (Lambda) DataCubeHelpers.parseQuery(code, false); + PureModelContextData data = DataCubeHelpers.injectNewFunction(pureModelContextData, lambda).getOne(); + Assert.assertEquals(expectedResult, objectMapper.writeValueAsString(DataCubeHelpers.getRelationReturnType(legendInterface, data))); + } + catch (IOException e) + { + throw new RuntimeException(e.getMessage()); + } + } + + @Test + public void testExtractRelationReturnTypeWithParserError() + { + String lambda = "|#>{test::TestDatabase.TEST0}#-->extend(~[newCol:c|'ok', colX: c|$c.FIRSTNAME])"; + testExtractRelationReturnTypeFailure("PARSER error at [1:31]: Unexpected token '-'. Valid alternatives: [';']", lambda); + } + + @Test + public void testExtractRelationReturnTypeWithCompilationError() + { + String lambda = "|#>{test::TestDatabase.TEST0}#->extend(~[newCol:c|'ok', colX: c|$c.FIRSTNAME2])"; + testExtractRelationReturnTypeFailure("COMPILATION error at [1:68-77]: The column 'FIRSTNAME2' can't be found in the relation (FIRSTNAME:String, LASTNAME:String)", lambda); + } + + private void testExtractRelationReturnTypeFailure(String errorMessage, String code) + { + EngineException e = Assert.assertThrows(EngineException.class, () -> + { + Lambda lambda = (Lambda) DataCubeHelpers.parseQuery(code, true); + PureModelContextData data = DataCubeHelpers.injectNewFunction(pureModelContextData, lambda).getOne(); + DataCubeHelpers.getRelationReturnType(legendInterface, data); + }); + Assert.assertEquals(errorMessage, EngineException.buildPrettyErrorMessage(e.getMessage(), e.getSourceInformation(), e.getErrorType())); + } +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestGridServer.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestGridServer.java deleted file mode 100644 index f017cb8025c..00000000000 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestGridServer.java +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.engine.repl; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.eclipse.collections.impl.factory.Lists; -import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser; -import org.finos.legend.engine.plan.execution.stores.relational.serialization.RelationalResultToJsonDefaultSerializer; -import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.application.AppliedFunction; -import org.finos.legend.engine.repl.core.legend.LegendInterface; -import org.finos.legend.engine.repl.core.legend.LocalLegendInterface; -import org.finos.legend.engine.repl.relational.httpServer.ReplGridServer; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; -import java.util.stream.Collectors; - -public class TestGridServer -{ - private String pmcd = "###Relational\n" + - "Database test::TestDatabase\n" + - "(\n" + - " Table TEST0\n" + - " (\n" + - " FIRSTNAME VARCHAR(200),\n" + - " LASTNAME VARCHAR(200)\n" + - " )\n" + - ")\n" + - "\n" + - "###Pure\n" + - "function a::b::c::d(): Any[*]\n" + - "{\n" + - " #>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(^meta::pure::mapping::Mapping(), test::test)\n" + - "}\n" + - "\n" + - "###Runtime\n" + - "Runtime test::test\n" + - "{\n" + - " mappings: [];\n" + - " connections:\n" + - " [\n" + - " test::TestDatabase:\n" + - " [ \n" + - " connection: test::TestConnection\n" + - " ]\n" + - "\n" + - " ];\n" + - "}\n" + - "\n" + - "\n" + - "###Connection\n" + - "RelationalDatabaseConnection test::TestConnection\n" + - "{\n" + - " store: test::TestDatabase;\n" + - " type: H2;\n" + - " specification: LocalH2\n" + - " {\n" + - " testDataSetupSqls: [\n" + - " '\\nDrop table if exists TEST0;\\nCreate Table TEST0(FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200));\\nInsert into TEST0 (FIRSTNAME, LASTNAME) values (\\'John\\', \\'Doe\\');\\nInsert into TEST0 (FIRSTNAME, LASTNAME) values (\\'Tim\\', \\'Smith\\');\\nInsert into TEST0 (FIRSTNAME, LASTNAME) values (\\'Nicole\\', \\'Doe\\');\\n\\n'\n" + - " ];\n" + - " };\n" + - " auth: DefaultH2;\n" + - "}"; - private LegendInterface legendInterface = new LocalLegendInterface(); - private ObjectMapper objectMapper = new ObjectMapper(); - private PureModelContextData pureModelContextData = legendInterface.parse(pmcd); - - @Test - public void testSort() - { - String lambda = "#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(^meta::pure::mapping::Mapping(), test::test)->sort([~FIRSTNAME->ascending()])"; - String expectedResult = "{\"builder\":{\"_type\":\"tdsBuilder\",\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"}]},\"activities\":[{\"_type\":\"relational\",\"sql\":\"select \\\"test0_0\\\".FIRSTNAME as \\\"FIRSTNAME\\\", \\\"test0_0\\\".LASTNAME as \\\"LASTNAME\\\" from TEST0 as \\\"test0_0\\\" where (\\\"test0_0\\\".FIRSTNAME <> 'Doe' OR \\\"test0_0\\\".FIRSTNAME is null) order by \\\"FIRSTNAME\\\"\"}],\"result\":{\"columns\":[\"FIRSTNAME\",\"LASTNAME\"],\"rows\":[{\"values\":[\"John\",\"Doe\"]},{\"values\":[\"Nicole\",\"Doe\"]},{\"values\":[\"Tim\",\"Smith\"]}]}}"; - test(expectedResult, lambda); - } - - @Test - public void testFilter() - { - String lambda = "#>{test::TestDatabase.TEST0}#->filter(c | ($c.FIRSTNAME != 'Doe' && $c.LASTNAME != 'Doe'))->from(^meta::pure::mapping::Mapping(), test::test)->sort([~FIRSTNAME->ascending()])"; - String expectedResult = "{\"builder\":{\"_type\":\"tdsBuilder\",\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"}]},\"activities\":[{\"_type\":\"relational\",\"sql\":\"select \\\"test0_0\\\".FIRSTNAME as \\\"FIRSTNAME\\\", \\\"test0_0\\\".LASTNAME as \\\"LASTNAME\\\" from TEST0 as \\\"test0_0\\\" where ((\\\"test0_0\\\".FIRSTNAME <> 'Doe' OR \\\"test0_0\\\".FIRSTNAME is null) and (\\\"test0_0\\\".LASTNAME <> 'Doe' OR \\\"test0_0\\\".LASTNAME is null)) order by \\\"FIRSTNAME\\\"\"}],\"result\":{\"columns\":[\"FIRSTNAME\",\"LASTNAME\"],\"rows\":[{\"values\":[\"Tim\",\"Smith\"]}]}}"; - test(expectedResult, lambda); - } - - @Test - public void testGroupBy() - { - String lambda = "#>{test::TestDatabase.TEST0}#->filter(c | $c.FIRSTNAME != 'Doe')->from(^meta::pure::mapping::Mapping(), test::test)->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()])"; - String expectedResult = "{\"builder\":{\"_type\":\"tdsBuilder\",\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"count\",\"type\":\"Integer\"}]},\"activities\":[{\"_type\":\"relational\",\"sql\":\"select \\\"test0_0\\\".FIRSTNAME as \\\"FIRSTNAME\\\", count(\\\"test0_0\\\".FIRSTNAME) as \\\"count\\\" from TEST0 as \\\"test0_0\\\" where (\\\"test0_0\\\".FIRSTNAME <> 'Doe' OR \\\"test0_0\\\".FIRSTNAME is null) group by \\\"FIRSTNAME\\\"\"}],\"result\":{\"columns\":[\"FIRSTNAME\",\"count\"],\"rows\":[{\"values\":[\"John\",1]},{\"values\":[\"Nicole\",1]},{\"values\":[\"Tim\",1]}]}}"; - test(expectedResult, lambda); - } - - @Test - public void testSlice() - { - String lambda = "#>{test::TestDatabase.TEST0}#->filter(c | ($c.FIRSTNAME != 'Doe' && $c.LASTNAME != 'Doe'))->from(^meta::pure::mapping::Mapping(), test::test)"; - String expectedResult = "{\"builder\":{\"_type\":\"tdsBuilder\",\"columns\":[{\"name\":\"FIRSTNAME\",\"type\":\"String\"},{\"name\":\"LASTNAME\",\"type\":\"String\"}]},\"activities\":[{\"_type\":\"relational\",\"sql\":\"select top 100 \\\"test0_0\\\".FIRSTNAME as \\\"FIRSTNAME\\\", \\\"test0_0\\\".LASTNAME as \\\"LASTNAME\\\" from TEST0 as \\\"test0_0\\\" where ((\\\"test0_0\\\".FIRSTNAME <> 'Doe' OR \\\"test0_0\\\".FIRSTNAME is null) and (\\\"test0_0\\\".LASTNAME <> 'Doe' OR \\\"test0_0\\\".LASTNAME is null))\"}],\"result\":{\"columns\":[\"FIRSTNAME\",\"LASTNAME\"],\"rows\":[{\"values\":[\"Tim\",\"Smith\"]}]}}"; - test(expectedResult, lambda, true); - } - - private void test(String expectedResult, String function) - { - test(expectedResult, function, false); - } - - private void test(String expectedResult, String function, boolean isPaginationEnabled) - { - try - { - Function originalFunction = (Function) pureModelContextData.getElements().stream().filter(e -> e.getPath().equals("a::b::c::d__Any_MANY_")).collect(Collectors.toList()).get(0); - ValueSpecification originalFunctionBody = originalFunction.body.get(0); - AppliedFunction currentFunction = (AppliedFunction) PureGrammarParser.newInstance().parseValueSpecification(function, null, 0, 0, true); - List newBody = Lists.mutable.of(currentFunction); - if (isPaginationEnabled) - { - ReplGridServer.applySliceFunction(newBody); - } - originalFunction.body = newBody; - String response = ReplGridServer.executeLambda(legendInterface, pureModelContextData, originalFunction, originalFunctionBody); - ReplGridServer.GridServerResult result = objectMapper.readValue(response, ReplGridServer.GridServerResult.class); - Assert.assertEquals(expectedResult, RelationalResultToJsonDefaultSerializer.removeComment(result.getResult())); - } - catch (IOException e) - { - throw new RuntimeException(e.getMessage()); - } - } -} diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/core/M3ParserGrammar.g4 b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/core/M3ParserGrammar.g4 index 1d830c5acd6..d15f55b25eb 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/core/M3ParserGrammar.g4 +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/core/M3ParserGrammar.g4 @@ -111,7 +111,7 @@ columnBuilders: TILDE (oneColSpec | colSpecArray) ; oneColSpec: identifier ((COLON (type | lambdaParam lambdaPipe) extraFunction? ))? ; -colSpecArray: (BRACKET_OPEN oneColSpec(COMMA oneColSpec)* BRACKET_CLOSE) +colSpecArray: (BRACKET_OPEN (oneColSpec(COMMA oneColSpec)*)? BRACKET_CLOSE) ; extraFunction: (COLON lambdaParam lambdaPipe) ; diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java index 064f3881e6c..0e9a5900df2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java @@ -21,7 +21,6 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType; import org.finos.legend.engine.pure.runtime.testConnection.CoreExternalTestConnectionCodeRepositoryProvider; import org.finos.legend.engine.test.shared.framework.TestServerResource; -import org.finos.legend.pure.code.core.CoreRelationalduckdbCodeRepositoryProvider; import org.finos.legend.pure.code.core.RelationCodeRepositoryProvider; import org.finos.legend.pure.m3.pct.reports.config.PCTReportConfiguration; import org.finos.legend.pure.m3.pct.reports.config.exclusion.ExclusionSpecification; @@ -36,10 +35,7 @@ public class Test_Relational_DuckDB_RelationFunctions_PCT extends PCTReportConfi private static final ReportScope reportScope = RelationCodeRepositoryProvider.relationFunctions; private static final Adapter adapter = CoreExternalTestConnectionCodeRepositoryProvider.duckDBAdapter; private static final String platform = "compiled"; - private static final MutableList expectedFailures = Lists.mutable.with( - // BUG: Column name with special characters is not properly escaped - one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "\"Unexpected error executing function with params [Anonymous_Lambda]\"") - ); + private static final MutableList expectedFailures = Lists.mutable.empty(); public static Test suite() { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/vendors/duckdb/DuckDBCommands.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/vendors/duckdb/DuckDBCommands.java index 4d754c800f3..064b3407771 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/vendors/duckdb/DuckDBCommands.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/vendors/duckdb/DuckDBCommands.java @@ -47,6 +47,12 @@ public String load(String tableName, String location) return "CREATE TABLE " + tableName + " AS SELECT * FROM read_csv('" + location + "', header=true);"; } + @Override + public String dropTable(String tableName, String location) + { + return "DROP TABLE " + tableName + ";"; + } + @Override public T accept(RelationalDatabaseCommandsVisitor visitor) { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure index 0be5f50aa7a..3af137b73a4 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure @@ -27,6 +27,7 @@ function <> meta::relational::functions::sqlQueryToString::duckD joinStringsProcessor = processJoinStringsOperationWithConcatCall_JoinStrings_1__SqlGenerationContext_1__String_1_, literalProcessor = $literalProcessor, selectSQLQueryProcessor = processSelectSQLQueryForDuckDB_SelectSQLQuery_1__SqlGenerationContext_1__Boolean_1__String_1_, + columnNameToIdentifier = columnNameToIdentifierForDuckDB_String_1__DbConfig_1__String_1_, identifierProcessor = processIdentifierWithDoubleQuotes_String_1__DbConfig_1__String_1_, dynaFuncDispatch = $dynaFuncDispatch, ddlCommandsTranslator = getDDLCommandsTranslator() @@ -220,6 +221,22 @@ function <> meta::relational::functions::sqlQueryToString::duckD '%s offset %s'->format([$format.separator, $s.fromRow->toOne()->getValueForTake($format, $dbConfig, $extensions)]) + if ($size == -1, | '', | ' limit %s'->format($size)); } +function <> meta::relational::functions::sqlQueryToString::duckDB::columnNameToIdentifierForDuckDB(columnName: String[1], dbConfig: DbConfig[1]): String[1] +{ + if( + $dbConfig.isDbReservedIdentifier($columnName->toLower()) || + // TODO: the way we handle quoting right now is not systematic, i.e. we allow so many way to interject custom logic + // as such, for duckdb, to keep it simple, we will not use the default implementation of converting column name to identifier + (!$columnName->startsWith('"') && ( + $columnName->contains(' ') || + $columnName->contains('/') || + $columnName->contains('|') + )), + |'"' + $columnName->toLower() + '"', + |$columnName + ); +} + function <> meta::relational::functions::sqlQueryToString::duckDB::duckDBReservedWords():String[*] { //https://github.com/duckdb/duckdb/blob/main/third_party/libpg_query/grammar/keywords/reserved_keywords.list diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/commands/RelationalDatabaseCommands.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/commands/RelationalDatabaseCommands.java index 2b8b8f66b65..4bdc40f8aae 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/commands/RelationalDatabaseCommands.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/commands/RelationalDatabaseCommands.java @@ -34,6 +34,11 @@ public String load(String tableName, String location) throw new RuntimeException("Load not implemented for " + this.getClass().getSimpleName()); } + public String dropTable(String tableName, String location) + { + throw new RuntimeException("Drop table not implemented for " + this.getClass().getSimpleName()); + } + public abstract IngestionMethod getDefaultIngestionMethod(); // public void buildTempTableFromResult(RelationalExecutionConfiguration config, Connection connection, StreamingResult result, String tableName) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java index 8756451cc58..7e3b4b5ac1f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java @@ -843,6 +843,18 @@ else if (c instanceof Double) { primitiveType = "Float"; } + else if (c instanceof Decimal) + { + primitiveType = "Decimal"; + } + else if (c instanceof Date) + { + primitiveType = "Date"; + } + else if (c instanceof Timestamp) + { + primitiveType = "DateTime"; + } else { throw new RuntimeException("Implement support for '" + c.getClass().getName() + "'");