Start splitting relational out. Extract external format from core Jav…

…a binding
finos · Feb 27, 2024 · 7b4c0cc · 7b4c0cc
1 parent 6d79d17
commit 7b4c0cc
Show file tree

Hide file tree

Showing 67 changed files with 838 additions and 193 deletions.
diff --git a/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml b/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml
@@ -154,6 +154,10 @@
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-xt-javaPlatformBinding-pure</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-javaPlatformBinding-externalFormat-pure</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-external-shared</artifactId>

diff --git a/...rc/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java b/...rc/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java
@@ -413,7 +413,7 @@ protected Iterable<? extends Class<? extends LegendPureCoreExtension>> getExpect
                 .with(CoreLegendPureCoreExtension.class)
                 .with(JSONLegendPureCoreExtension.class)
                 .with(RelationalLegendPureCoreExtension.class)
-                .with(BindingJavaBindingLegendPureCoreExtension.class)
+                .with(ExternalFormatJavaBindingLegendPureCoreExtension.class)
                 .with(M2MJavaBindingLegendPureCoreExtension.class)
                 .with(ServiceStoreJavaBindingLegendPureCoreExtension.class)
                 .with(ServiceStoreLegendPureCoreExtension.class)
@@ -530,6 +530,7 @@ protected Iterable<String> getExpectedCodeRepositories()
                 .with("core_text_metamodel")
                 .with("core_external_language_java")
                 .with("core_java_platform_binding")
+                .with("core_java_platform_binding_external_format")
                 .with("core_relational_java_platform_binding")
                 .with("core_servicestore_java_platform_binding")
                 .with("core_external_format_flatdata_java_platform_binding")

diff --git a/legend-engine-config/legend-engine-pure-code-compiled-core-configuration/pom.xml b/legend-engine-config/legend-engine-pure-code-compiled-core-configuration/pom.xml
@@ -299,6 +299,10 @@
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-xt-javaPlatformBinding-pure</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-javaPlatformBinding-externalFormat-pure</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-xt-relationalStore-pure</artifactId>

diff --git a/...re-code-compiled-core-configuration/src/main/resources/core_configuration.definition.json b/...re-code-compiled-core-configuration/src/main/resources/core_configuration.definition.json
@@ -8,6 +8,7 @@
     "core_service",
     "core_external_language_java",
     "core_java_platform_binding",
+    "core_java_platform_binding_external_format",
     "core_relational",
     "core_relational_java_platform_binding",
     "core_servicestore",

diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/pom.xml b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/pom.xml
@@ -105,10 +105,10 @@
             <groupId>org.finos.legend.pure</groupId>
             <artifactId>legend-pure-m2-store-relational-pure</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.finos.legend.engine</groupId>
-            <artifactId>legend-engine-xt-relationalStore-executionPlan</artifactId>
-        </dependency>
+<!--        <dependency>-->
+<!--            <groupId>org.finos.legend.engine</groupId>-->
+<!--            <artifactId>legend-engine-xt-relationalStore-executionPlan</artifactId>-->
+<!--        </dependency>-->
         <dependency>
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-xt-relationalStore-grammar</artifactId>

diff --git a/...l/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java b/...l/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java
@@ -14,18 +14,17 @@
 
 package org.finos.legend.engine.repl.client;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.list.MutableList;
 import org.eclipse.collections.impl.block.predicate.checked.CheckedPredicate;
-import org.finos.legend.engine.plan.execution.stores.StoreType;
-import org.finos.legend.engine.plan.execution.stores.relational.plugin.RelationalStoreExecutor;
-import org.finos.legend.engine.repl.client.commands.*;
+import org.finos.legend.engine.repl.core.Command;
+import org.finos.legend.engine.repl.core.ReplExtension;
+import org.finos.legend.engine.repl.core.commands.*;
 import org.finos.legend.engine.repl.client.jline3.JLine3Completer;
 import org.finos.legend.engine.repl.client.jline3.JLine3Highlighter;
-import org.finos.legend.engine.repl.REPLInterface;
+import org.finos.legend.engine.repl.core.legend.LegendInterface;
 import org.finos.legend.engine.repl.client.jline3.JLine3Parser;
-import org.finos.legend.engine.repl.LocalREPL;
+import org.finos.legend.engine.repl.core.legend.LocalLegendInterface;
 import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
 import org.jline.reader.LineReader;
 import org.jline.reader.LineReaderBuilder;
@@ -34,58 +33,42 @@
 import org.jline.utils.AttributedStringBuilder;
 import org.jline.utils.AttributedStyle;
 
-import java.sql.*;
-
-import static org.finos.legend.engine.repl.database.MetadataReader.getTables;
-
 public class Client
 {
-    public static final REPLInterface replInterface = new LocalREPL();
-
-    public static MutableList<String> state = Lists.mutable.empty();
-
-    public static Terminal terminal;
+    private final LegendInterface legendInterface = new LocalLegendInterface();
+    private final Terminal terminal;
+    private final LineReader reader;
+    private boolean debug = false;
 
-    public static boolean debug = false;
-
-    public static ObjectMapper objectMapper = new ObjectMapper();
-
-    static
-    {
-        System.setProperty("legend.test.h2.port", "1975");
-    }
+    private MutableList<ReplExtension> replExtensions = Lists.mutable.empty();
 
     public static void main(String[] args) throws Exception
     {
         new Client().loop();
     }
 
-    public LineReader reader;
-
     public MutableList<Command> commands;
 
     public Client() throws Exception
     {
-        terminal = TerminalBuilder.terminal();
+        this.terminal = TerminalBuilder.terminal();
 
-        terminal.writer().println("\n" + Logos.logos.get((int) (Logos.logos.size() * Math.random())) + "\n");
+        this.terminal.writer().println("\n" + Logos.logos.get((int) (Logos.logos.size() * Math.random())) + "\n");
 
-        this.commands = Lists.mutable.with(new DB(this), new Ext(this), new Debug(this), new Graph(this), new Load(this), new Execute(this));
+        this.commands = replExtensions.flatCollect(ReplExtension::getExtraCommands).withAll(Lists.mutable.with(new Ext(this), new Debug(this), new Graph(this), new Execute(this)));
         this.commands.add(0, new Help(this, this.commands));
 
-        reader = LineReaderBuilder.builder()
+        this.reader = LineReaderBuilder.builder()
                 .terminal(terminal)
                 .highlighter(new JLine3Highlighter())
                 .parser(new JLine3Parser())//new DefaultParser().quoteChars(new char[]{'"'}))
                 .completer(new JLine3Completer(this.commands))
                 .build();
 
-        terminal.writer().println("Warming up...");
-        terminal.flush();
+        this.terminal.writer().println("Warming up...");
+        this.terminal.flush();
         ((Execute) this.commands.getLast()).execute("1+1");
-        terminal.writer().println("Ready!\n");
-
-
+        this.terminal.writer().println("Ready!\n");
     }
 
     public void loop()
@@ -102,7 +85,7 @@ public void loop()
 
             try
             {
-                commands.detect(new CheckedPredicate<Command>()
+                this.commands.detect(new CheckedPredicate<Command>()
                 {
                     @Override
                     public boolean safeAccept(Command c) throws Exception
@@ -117,16 +100,16 @@ public boolean safeAccept(Command c) throws Exception
             }
             catch (Exception ee)
             {
-                terminal.writer().println(ee.getMessage());
-                if (debug)
+                this.terminal.writer().println(ee.getMessage());
+                if (this.debug)
                 {
                     ee.printStackTrace();
                 }
             }
         }
     }
 
-    public static void printError(EngineException e, String line)
+    public void printError(EngineException e, String line)
     {
         int e_start = e.getSourceInformation().startColumn;
         int e_end = e.getSourceInformation().endColumn;
@@ -142,53 +125,47 @@ public static void printError(EngineException e, String line)
             ab.append(mid);
             ab.style(new AttributedStyle().underlineOff().boldOff().foreground(0, 200, 0));
             ab.append(end);
-            terminal.writer().println("");
-            terminal.writer().println(ab.toAnsi());
+            this.terminal.writer().println("");
+            this.terminal.writer().println(ab.toAnsi());
         }
-        terminal.writer().println(e.getMessage());
-        if (debug)
+        this.terminal.writer().println(e.getMessage());
+        if (this.debug)
         {
             e.printStackTrace();
         }
     }
 
-    public Connection getConnection()
+    public Terminal getTerminal()
     {
-        RelationalStoreExecutor r = (RelationalStoreExecutor) ((Execute) this.commands.getLast()).planExecutor.getExecutorsOfType(StoreType.Relational).getFirst();
-        return r.getStoreState().getRelationalExecutor().getConnectionManager().getTestDatabaseConnection();
+        return this.terminal;
     }
 
-    public MutableList<String> buildState()
+    public LegendInterface getLegendInterface()
+    {
+        return this.legendInterface;
+    }
+
+    public boolean isDebug()
     {
-        MutableList<String> res = Lists.mutable.withAll(state);
-
-        res.add("###Relational\n" +
-                "Database test::TestDatabase" +
-                "(" +
-                getTables(getConnection()).collect(table -> "Table " + table.name + "(" + table.columns.collect(c -> (c.name.contains(" ") ? "\"" + c.name + "\"" : c.name) + " " + c.type).makeString(",") + ")").makeString("\n") +
-                ")\n");
-
-        res.add("###Connection\n" +
-                "RelationalDatabaseConnection test::TestConnection\n" +
-                "{\n" +
-                "   store: test::TestDatabase;" +
-                "   specification: LocalH2{};" +
-                "   type: H2;" +
-                "   auth: DefaultH2;" +
-                "}\n");
-
-        res.add("###Runtime\n" +
-                "Runtime test::TestRuntime\n" +
-                "{\n" +
-                "   mappings : [];" +
-                "   connections:\n" +
-                "   [\n" +
-                "       test::TestDatabase : [connection: test::TestConnection]\n" +
-                "   ];\n" +
-                "}\n");
+        return this.debug;
+    }
 
-        return res;
+    public void setDebug(boolean debug)
+    {
+        this.debug = debug;
     }
 
+    public MutableList<ReplExtension> getReplExtensions()
+    {
+        return replExtensions;
+    }
 
+    public MutableList<String> buildState()
+    {
+        MutableList<String> res = Lists.mutable.empty();
+
+        res.addAll(this.replExtensions.flatCollect(ReplExtension::getExtraState));
+
+        return res;
+    }
 }
diff --git a/...repl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Completer.java b/...repl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Completer.java
@@ -15,7 +15,7 @@
 package org.finos.legend.engine.repl.client.jline3;
 
 import org.eclipse.collections.api.list.MutableList;
-import org.finos.legend.engine.repl.client.Command;
+import org.finos.legend.engine.repl.core.Command;
 import org.jline.reader.Candidate;
 import org.jline.reader.Completer;
 import org.jline.reader.LineReader;

diff --git a/...os/legend/engine/repl/client/Command.java → ...inos/legend/engine/repl/core/Command.java b/...os/legend/engine/repl/client/Command.java → ...inos/legend/engine/repl/core/Command.java
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.finos.legend.engine.repl.client;
+package org.finos.legend.engine.repl.core;
 
 import org.eclipse.collections.api.list.MutableList;
 import org.jline.reader.Candidate;

diff --git a/...end-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/ReplExtension.java b/...end-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/ReplExtension.java
@@ -0,0 +1,49 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.engine.repl.core;
+
+import org.eclipse.collections.api.list.MutableList;
+import org.eclipse.collections.impl.factory.Lists;
+import org.finos.legend.engine.plan.execution.result.Result;
+import org.finos.legend.engine.shared.core.extension.LegendExtension;
+
+public interface ReplExtension extends LegendExtension
+{
+    @Override
+    default MutableList<String> group()
+    {
+        return Lists.mutable.with("REPL");
+    }
+
+    @Override
+    default String type()
+    {
+        return "core";
+    }
+
+    @Override
+    default MutableList<String> typeGroup()
+    {
+        return Lists.mutable.empty();
+    }
+
+    MutableList<Command> getExtraCommands();
+
+    MutableList<String> getExtraState();
+
+    boolean supports(Result res);
+
+    String print(Result res);
+}
diff --git a/...nd/engine/repl/client/commands/Debug.java → ...gend/engine/repl/core/commands/Debug.java b/...nd/engine/repl/client/commands/Debug.java → ...gend/engine/repl/core/commands/Debug.java
@@ -12,18 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.finos.legend.engine.repl.client.commands;
+package org.finos.legend.engine.repl.core.commands;
 
 import org.eclipse.collections.api.list.MutableList;
 import org.finos.legend.engine.repl.client.Client;
-import org.finos.legend.engine.repl.client.Command;
+import org.finos.legend.engine.repl.core.Command;
 import org.jline.reader.Candidate;
 import org.jline.reader.LineReader;
 import org.jline.reader.ParsedLine;
 
 public class Debug implements Command
 {
-    private Client client;
+    private final Client client;
 
     public Debug(Client client)
     {
@@ -44,13 +44,13 @@ public boolean process(String line) throws Exception
             String[] cmd = line.split(" ");
             if (cmd.length == 1)
             {
-                client.debug = !client.debug;
+                this.client.setDebug(!this.client.isDebug());
             }
             else
             {
-                client.debug = Boolean.parseBoolean(cmd[1]);
+                this.client.setDebug(Boolean.parseBoolean(cmd[1]));
             }
-            client.terminal.writer().println("debug: " + client.debug);
+            this.client.getTerminal().writer().println("debug: " + this.client.isDebug());
             return true;
         }
         return false;