repl: add datacube commands (#2959)

* repl: infrastructure API for datacube * repl: support debugging for datacube query execution * repl: standardize display for errors and align with color palette of terminal * repl: fix problem with command ordering causing execute command to take over all others * repl: improve execute, cache and add datacube commands * repl: separate data-cube code to its own extension * repl: build shaded JAR for REPL with DataCube * repl: add DEBUG commands for DataCube extension * fix modules' versions post rebase * repl: adjust data-cube compiler usage post rebase * repl: fix a bug where exclamation point (NOT operator) crash auto-completer * repl: support persisting command history to file * repl: simplify cache commands * repl: improve printing relational result (with truncation and metadata) * repl: standardize terminal print command usage * repl: make sure Show command warn about invalid last Execute command * datacube: return query code as part of execution call * datacube: bump datacube client [email protected]
finos · Jul 31, 2024 · 680b52a · 680b52a
1 parent 817648f
commit 680b52a
Show file tree

Hide file tree

Showing 60 changed files with 2,164 additions and 784 deletions.
diff --git a/legend-engine-config/legend-engine-repl/README.md b/legend-engine-config/legend-engine-repl/README.md
@@ -16,7 +16,7 @@ load data.csv local::DuckDuckConnection test1
 # then show the data
 #>{local::DuckDuckDatabase.test1}#->sort([])->from(local::DuckDuckRuntime)
 
-# to show the result grid in GUI mode
+# to show the result grid in DataCube
 show
 # to debug when error occurs, toggle debug mode
 debug
@@ -50,7 +50,7 @@ The exact command to run it can be copied from the command run by IntelliJ (and
 
 ```shell
 # NOTE: this command has a very long classpath!
-java -Dfile.encoding=UTF-8 -classpath ... -Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5005 org.finos.legend.engine.repl.relational.client.RClient
+java -Dfile.encoding=UTF-8 -classpath ... -Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5005 org.finos.legend.engine.repl.dataCube.client.DataCubeClient
 ```
 
 Then, in IntelliJ, start the Remote JVM Debug session with the specified port (5005 like in command above).
@@ -68,5 +68,5 @@ Or if in a terminal
 
 ```shell
 # NOTE: this command has a very long classpath!
-java -Dfile.encoding=UTF-8 -classpath ... -Dlegend.repl.devPort=9006 -Dlegend.repl.devWebAppBaseUrl=http://localhost:9005 org.finos.legend.engine.repl.relational.client.RClient
+java -Dfile.encoding=UTF-8 -classpath ... -Dlegend.repl.devPort=9006 -Dlegend.repl.devWebAppBaseUrl=http://localhost:9005 org.finos.legend.engine.repl.dataCube.client.DataCubeClient
 ```
diff --git a/legend-engine-config/legend-engine-repl/docs/repl-debug-setup.png b/legend-engine-config/legend-engine-repl/docs/repl-debug-setup.png
diff --git a/legend-engine-config/legend-engine-repl/docs/repl-webapp-dev-setup.png b/legend-engine-config/legend-engine-repl/docs/repl-webapp-dev-setup.png
diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/pom.xml b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/pom.xml
@@ -170,6 +170,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>org.jline</groupId>

diff --git a/...engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java b/...engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java
@@ -61,8 +61,8 @@
 import java.util.List;
 import java.util.Objects;
 
-import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_QUALIFIED_PATH;
-import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_SIGNATURE;
+import static org.finos.legend.engine.repl.shared.ExecutionHelper.REPL_RUN_FUNCTION_QUALIFIED_PATH;
+import static org.finos.legend.engine.repl.shared.ExecutionHelper.REPL_RUN_FUNCTION_SIGNATURE;
 
 public class Completer
 {

diff --git a/...l/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java b/...l/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java
@@ -14,6 +14,8 @@
 
 package org.finos.legend.engine.repl.client;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.io.FileUtils;
 import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.list.MutableList;
 import org.eclipse.collections.impl.block.predicate.checked.CheckedPredicate;
@@ -27,6 +29,7 @@
 import org.finos.legend.engine.repl.core.commands.*;
 import org.finos.legend.engine.repl.core.legend.LegendInterface;
 import org.finos.legend.engine.repl.core.legend.LocalLegendInterface;
+import org.finos.legend.engine.shared.core.ObjectMapperFactory;
 import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
 import org.jline.reader.EndOfFileException;
 import org.jline.reader.LineReader;
@@ -37,11 +40,15 @@
 import org.jline.utils.AttributedStringBuilder;
 import org.jline.utils.AttributedStyle;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
+
 import static org.jline.jansi.Ansi.ansi;
 import static org.jline.reader.LineReader.BLINK_MATCHING_PAREN;
 
 public class Client
 {
+    private final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports();
     private final LegendInterface legendInterface = new LocalLegendInterface();
     private final Terminal terminal;
     private final LineReader reader;
@@ -66,26 +73,34 @@ public Client(MutableList<ReplExtension> replExtensions, MutableList<CompleterEx
         this.state = new ModelState(this.legendInterface, this.replExtensions);
         this.terminal = TerminalBuilder.terminal();
 
+        this.initialize();
         replExtensions.forEach(e -> e.initialize(this));
 
-        this.terminal.writer().println(ansi().fgBrightBlack().a("Welcome to the Legend REPL! Press 'Enter' or type 'help' to see the list of available commands.").reset());
-        this.terminal.writer().println("\n" + Logos.logos.get((int) (Logos.logos.size() * Math.random())) + "\n");
+        this.printDebug("Welcome to the Legend REPL! Press 'Enter' or type 'help' to see the list of available commands.");
+        this.printInfo("\n" + Logos.logos.get((int) (Logos.logos.size() * Math.random())) + "\n");
 
+        // NOTE: the order here matters, the default command 'help' should always go first
+        // and "catch-all" command 'execute' should always go last
         this.commands = replExtensions
                 .flatCollect(ReplExtension::getExtraCommands)
                 .withAll(
                         Lists.mutable.with(
                                 new Ext(this),
                                 new Debug(this),
                                 new Graph(this),
-                                new Execute(this, planExecutor)
+                                new Execute(this)
                         )
                 );
 
         this.commands.add(0, new Help(this, this.commands));
 
         this.reader = LineReaderBuilder.builder()
                 .terminal(terminal)
+                // Configure history file
+                // See https://github.com/jline/jline3/wiki/History
+                .variable(LineReader.HISTORY_FILE, this.getHomeDir().resolve("history"))
+                .variable(LineReader.HISTORY_FILE_SIZE, 1_000)
+                .variable(LineReader.HISTORY_IGNORE, ": *") // make sure empty space(s) are not persisted
                 // Disable cursor jumping to opening brace when typing closing brace
                 // See https://github.com/jline/jline3/issues/216
                 .variable(BLINK_MATCHING_PAREN, false)
@@ -95,16 +110,19 @@ public Client(MutableList<ReplExtension> replExtensions, MutableList<CompleterEx
                 .option(LineReader.Option.INSERT_TAB, true)
                 // Make sure word navigation works properly with Alt + (left/right) arrow key
                 .variable(LineReader.WORDCHARS, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-$")
+                // Make sure to not break the completer when exclamation sign is present
+                // Do this by disabling history expansion
+                // See https://github.com/jline/jline3/issues/246
+                .option(LineReader.Option.DISABLE_EVENT_EXPANSION, true)
                 .highlighter(new JLine3Highlighter())
                 .parser(new JLine3Parser())
                 .completer(new JLine3Completer(this.commands))
                 .build();
 
-        this.terminal.writer().println("Warming up...");
+        this.printInfo("Warming up...");
         this.terminal.flush();
         ((Execute) this.commands.getLast()).execute("1+1");
-        this.terminal.writer().println("Ready!\n");
-
+        this.printInfo("Ready!\n");
     }
 
     public void loop()
@@ -116,6 +134,8 @@ public void loop()
                 String line = this.reader.readLine("> ");
                 if (line == null || line.equalsIgnoreCase("exit"))
                 {
+                    System.exit(0);
+                    this.persistHistory();
                     break;
                 }
 
@@ -126,13 +146,24 @@ public void loop()
                     @Override
                     public boolean safeAccept(Command c) throws Exception
                     {
-                        return c.process(line);
+                        try
+                        {
+                            return c.process(line);
+                        }
+                        catch (RuntimeException e)
+                        {
+                            throw e;
+                        }
+                        catch (Exception e)
+                        {
+                            throw new RuntimeException(e);
+                        }
                     }
                 });
             }
             catch (EngineException e)
             {
-                printError(e, this.reader.getBuffer().toString());
+                printEngineError(e, this.reader.getBuffer().toString());
             }
             // handle Ctrl + C: if the input is not empty, start a new line; otherwise, exit
             catch (UserInterruptException e)
@@ -141,6 +172,7 @@ public boolean safeAccept(Command c) throws Exception
                 if (lineContent.isEmpty())
                 {
                     System.exit(0);
+                    this.persistHistory();
                     break;
                 }
                 else
@@ -152,45 +184,105 @@ public boolean safeAccept(Command c) throws Exception
             catch (EndOfFileException e)
             {
                 System.exit(0);
+                this.persistHistory();
                 break;
             }
             catch (Exception e)
             {
-                this.terminal.writer().println(ansi().fgRed().a(e.getMessage()).reset());
+                this.printError(e.getMessage());
                 if (this.debug)
                 {
                     e.printStackTrace();
                 }
             }
+            finally
+            {
+                this.persistHistory();
+            }
         }
     }
 
-    public void printError(EngineException e, String line)
+    public void printEngineError(EngineException e, String line)
     {
         int e_start = e.getSourceInformation().startColumn;
         int e_end = e.getSourceInformation().endColumn;
-        if (e_start <= line.length())
+        try
+        {
+            if (e_start <= line.length())
+            {
+                String beg = line.substring(0, e_start - 1);
+                String mid = line.substring(e_start - 1, e_end);
+                String end = line.substring(e_end, line.length());
+                AttributedStringBuilder ab = new AttributedStringBuilder();
+                ab.style(new AttributedStyle().underlineOff().foregroundOff());
+                ab.append(beg);
+                ab.style(new AttributedStyle().underline().foreground(AttributedStyle.RED));
+                ab.append(mid);
+                ab.style(new AttributedStyle().underlineOff().foregroundOff());
+                ab.append(end);
+                this.printInfo("");
+                this.printInfo(ab.toAnsi());
+            }
+        }
+        catch (Exception ex)
         {
-            String beg = line.substring(0, e_start - 1);
-            String mid = line.substring(e_start - 1, e_end);
-            String end = line.substring(e_end, line.length());
-            AttributedStringBuilder ab = new AttributedStringBuilder();
-            ab.style(new AttributedStyle().underlineOff().boldOff().foreground(0, 200, 0));
-            ab.append(beg);
-            ab.style(new AttributedStyle().underline().bold().foreground(200, 0, 0));
-            ab.append(mid);
-            ab.style(new AttributedStyle().underlineOff().boldOff().foreground(0, 200, 0));
-            ab.append(end);
-            this.terminal.writer().println("");
-            this.terminal.writer().println(ab.toAnsi());
+            // do nothing
         }
-        this.terminal.writer().println(e.getMessage());
+        this.printError(e.getMessage());
         if (this.debug)
         {
             e.printStackTrace();
         }
     }
 
+    public void printDebug(String message)
+    {
+        this.terminal.writer().println(ansi().fgBrightBlack().a(message).reset());
+    }
+
+    public void printInfo(String message)
+    {
+        this.terminal.writer().println(message);
+    }
+
+    public void printError(String message)
+    {
+        this.terminal.writer().println(ansi().fgRed().a(message).reset());
+    }
+
+    private void initialize()
+    {
+        try
+        {
+            Path homeDir = this.getHomeDir();
+            if (Files.notExists(homeDir))
+            {
+                Files.createDirectories(homeDir);
+            }
+        }
+        catch (Exception e)
+        {
+            this.printError("Failed to create home directory at: " + this.getHomeDir().toString());
+        }
+    }
+
+    private void persistHistory()
+    {
+        try
+        {
+            this.reader.getHistory().save();
+        }
+        catch (Exception e)
+        {
+            // ignore
+        }
+    }
+
+    public Path getHomeDir()
+    {
+        return FileUtils.getUserDirectory().toPath().resolve(".legend/repl");
+    }
+
     public Terminal getTerminal()
     {
         return this.terminal;
@@ -231,8 +323,25 @@ public MutableList<CompleterExtension> getCompleterExtensions()
         return this.completerExtensions;
     }
 
-    public Execute getExecuteCommand()
+    public ObjectMapper getObjectMapper()
+    {
+        return objectMapper;
+    }
+
+    public String getLastCommand()
     {
-        return (Execute) this.commands.detect(c -> c instanceof Execute);
+        return this.getLastCommand(0);
+    }
+
+    public String getLastCommand(int skip)
+    {
+        try
+        {
+            return this.reader.getHistory().get(this.reader.getHistory().last() - skip);
+        }
+        catch (Exception e)
+        {
+            return null;
+        }
     }
 }
diff --git a/...pl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Highlighter.java b/...pl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Highlighter.java
@@ -46,7 +46,7 @@ public void setErrorIndex(int i)
 
     public static void drawCommand(AttributedStringBuilder ab, String command)
     {
-        ab.style(new AttributedStyle().foreground(0, 200, 0).italic());
+        ab.style(new AttributedStyle().foreground(AttributedStyle.GREEN).italic());
         ab.append(command);
     }
 }
diff --git a/...pl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Command.java b/...pl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Command.java
@@ -21,14 +21,19 @@
 
 public interface Command
 {
-    public boolean process(String cmd) throws Exception;
+    boolean process(String cmd) throws Exception;
 
-    public String documentation();
+    String documentation();
 
-    public default String description()
+    default String description()
     {
         return "";
     }
 
-    public MutableList<Candidate> complete(String cmd, LineReader lineReader, ParsedLine parsedLine);
+    default Command parentCommand()
+    {
+        return null;
+    }
+
+    MutableList<Candidate> complete(String cmd, LineReader lineReader, ParsedLine parsedLine);
 }