Schema support in relation accessor

.gs-project.yml

@@ -1 +1 @@
-productGuid: "product::315850"
+productGuid: "product::776807"

legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/parser/ParserFixer.java

@@ -207,13 +207,13 @@ else if (buffer)
             }
             if (content.contains("{") && !content.contains("}"))
             {
-                return value + "}#";
+                return (value.contains(magicToken) ? value : value + magicToken) + "}#";
             }
             if (content.contains("{") && content.contains("}"))
             {
                 return value + "#";
             }
-            return value + "{}#";
+            return value + "{" + magicToken + "}#";
         }
         return value;
 

legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/test/java/org/finos/legend/engine/repl/TestParserFixer.java

@@ -37,11 +37,11 @@ public void testArithmetic()
     public void testIsland()
     {
         Assert.assertEquals("#MaGiCToKeN{}#", fixCode("#"));
-        Assert.assertEquals("#>{}#", fixCode("#>"));
-        Assert.assertEquals("#>{}#", fixCode("#>{"));
+        Assert.assertEquals("#>{MaGiCToKeN}#", fixCode("#>"));
+        Assert.assertEquals("#>{MaGiCToKeN}#", fixCode("#>{"));
         Assert.assertEquals("#>{}#", fixCode("#>{}"));
         Assert.assertEquals("#>{a::A.t}#->fil()", fixCode("#>{a::A.t}#->fil"));
-        Assert.assertEquals("#>{a::A.t}#->x(#>{}#)", fixCode("#>{a::A.t}#->x(#>"));
+        Assert.assertEquals("#>{a::A.t}#->x(#>{MaGiCToKeN}#)", fixCode("#>{a::A.t}#->x(#>"));
         Assert.assertEquals("#>{a::A.t}#->x(#>{a::A.MaGiCToKeN}#)", fixCode("#>{a::A.t}#->x(#>{a::A."));
     }
 

legend-engine-config/legend-engine-repl/legend-engine-repl-relational/pom.xml

@@ -31,11 +31,7 @@
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-repl-client</artifactId>
         </dependency>
-
-        <dependency>
-            <groupId>org.finos.legend.pure</groupId>
-            <artifactId>legend-pure-m4</artifactId>
-        </dependency>
+
         <dependency>
             <groupId>org.finos.legend.pure</groupId>
             <artifactId>legend-pure-m3-core</artifactId>
@@ -151,5 +147,13 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-protocol</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-grammar</artifactId>
+        </dependency>
     </dependencies>
 </project>

legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/autocomplete/RelationalCompleterExtension.java

@@ -15,16 +15,22 @@
 package org.finos.legend.engine.repl.relational.autocomplete;
 
 import org.eclipse.collections.api.list.MutableList;
+import org.eclipse.collections.api.set.SetIterable;
 import org.eclipse.collections.impl.factory.Lists;
 import org.eclipse.collections.impl.utility.ListIterate;
+import org.finos.legend.engine.language.pure.compiler.toPureGraph.HelperRelationalBuilder;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
+import org.finos.legend.engine.protocol.pure.v1.model.SourceInformation;
 import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.relation.RelationStoreAccessor;
 import org.finos.legend.engine.repl.autocomplete.CompleterExtension;
 import org.finos.legend.engine.repl.autocomplete.CompletionItem;
 import org.finos.legend.engine.repl.autocomplete.CompletionResult;
 import org.finos.legend.engine.repl.autocomplete.parser.ParserFixer;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store;
+import org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database;
+import org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Schema;
+import org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.relation.Table;
 
 public class RelationalCompleterExtension implements CompleterExtension
 {
@@ -33,32 +39,57 @@ public CompletionResult extraClassInstanceProcessor(Object islandExpr, PureModel
     {
         if (islandExpr instanceof RelationStoreAccessor)
         {
-            MutableList<String> path = Lists.mutable.withAll(((RelationStoreAccessor) islandExpr).path);
-            String writtenPath = path.makeString("::").replace(ParserFixer.magicToken, "");
-            MutableList<Store> elements = pureModel.getAllStores().select(c -> nameMatch(c, writtenPath)).toList();
-            if (elements.size() == 1 &&
-                    writtenPath.startsWith(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(elements.get(0))) &&
-                    !writtenPath.equals(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(elements.get(0)))
-            )
+            RelationStoreAccessor relationStoreAccessor = (RelationStoreAccessor) islandExpr;
+            MutableList<String> path = Lists.adapt(relationStoreAccessor.path);
+
+            if (path.anySatisfy(x -> x.isEmpty() || x.contains(ParserFixer.magicToken)))
             {
-                org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database db = (org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database) elements.get(0);
-                String tableName = writtenPath.replace(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(db), "").replace("::", "");
-                MutableList<? extends org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.relation.Table> tables = db._schemas().isEmpty() ? Lists.mutable.empty() : db._schemas().getFirst()._tables().toList();
-                MutableList<? extends org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.relation.Table> foundTables = tables.select(c -> c._name().startsWith(tableName));
-                if ((foundTables.size() == 1 && foundTables.get(0)._name().equals(path.getLast())))
+                String writtenPath = path.get(0).replace(ParserFixer.magicToken, "");
+                MutableList<Store> elements = pureModel.getAllStores().select(c -> nameMatch(c, writtenPath)).toList();
+
+                MutableList<CompletionItem> completionItems = Lists.mutable.empty();
+
+                if (elements.size() == 1 && path.size() > 1)
                 {
-                    return new CompletionResult(Lists.mutable.empty());
+                    org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database db = (org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database) elements.get(0);
+
+                    if (path.size() < 3)
+                    {
+                        String tableOrSchema = path.get(1).replace(ParserFixer.magicToken, "").replace("::", "");
+                        completionItems.addAll(getTableSuggestions(db, "default", tableOrSchema));
+                        completionItems.addAll(getSchemaSuggestions(db, tableOrSchema));
+                    }
+                    else
+                    {
+                        String schema = path.get(1);
+                        String tableName = path.get(2).replace(ParserFixer.magicToken, "").replace("::", "");
+                        completionItems.addAll(getTableSuggestions(db, schema, tableName));
+                    }
                 }
                 else
                 {
-                    return new CompletionResult(foundTables.collect(c -> new CompletionItem(c._name(), c._name() + "}")));
+                    ListIterate.collect(elements, c -> new CompletionItem(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c), ">{" + org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c) + '.'), completionItems);
                 }
+
+                return new CompletionResult(completionItems);
             }
-            return new CompletionResult(ListIterate.collect(elements, c -> new CompletionItem(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c), ">{" + org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c))).toList());
         }
         return null;
     }
 
+    private static MutableList<CompletionItem> getSchemaSuggestions(Database db, String schemaName)
+    {
+        MutableList<? extends Schema> foundSchemas = db._schemas().select(schema -> !schema._name().equals("default") && schema._name().startsWith(schemaName)).toSortedListBy(Schema::_name);
+        return foundSchemas.collect(c -> new CompletionItem(c._name(), c._name() + "."));
+    }
+
+    private static MutableList<CompletionItem> getTableSuggestions(Database db, String schemaName, String tableName)
+    {
+        SetIterable<Table> tables = HelperRelationalBuilder.getAllTablesInSchema(db, schemaName, SourceInformation.getUnknownSourceInformation());
+        MutableList<? extends Table> foundTables = tables.select(c -> c._name().startsWith(tableName)).toSortedListBy(Table::_name);
+        return foundTables.collect(c -> new CompletionItem(c._name(), c._name() + "}#"));
+    }
+
     private static boolean nameMatch(PackageableElement c, String writtenPath)
     {
         String path = org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c);

legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java

@@ -26,12 +26,16 @@ public class TestCompleter
     @Test
     public void testRelationAccessor()
     {
-        Assert.assertEquals("[a::A , >{a::A]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>")));
-        Assert.assertEquals("[a::other , >{a::other], [a::ABC , >{a::ABC]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a")));
-        Assert.assertEquals("[a::ABC , >{a::ABC]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A")));
-        Assert.assertEquals("[a::ABC , >{a::ABC]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC")));
-        Assert.assertEquals("[t , t}]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.")));
-        Assert.assertEquals("[tab , tab}]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table co(val INTEGER) Table tab(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t")));
+        Assert.assertEquals("[a::A , >{a::A.]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>")));
+        Assert.assertEquals("[a::other , >{a::other.], [a::ABC , >{a::ABC.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a")));
+        Assert.assertEquals("[a::ABC , >{a::ABC.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A")));
+        Assert.assertEquals("[a::ABC , >{a::ABC.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC")));
+        Assert.assertEquals("[t , t}#]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.")));
+        Assert.assertEquals("[at , at}#], [t , t}#], [ts , ts.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Schema ts(Table t(col VARCHAR(200))) Table t(col VARCHAR(200)) Table at(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.")));
+        Assert.assertEquals("[t , t}#], [ts , ts.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Schema ts(Table t(col VARCHAR(200))) Table t(col VARCHAR(200)) Table at(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.t")));
+        Assert.assertEquals("[ts , ts.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Schema ts(Table t(col VARCHAR(200))) Table t(col VARCHAR(200)) Table at(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.ts")));
+        Assert.assertEquals("[t , t}#]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Schema ts(Table t(col VARCHAR(200))) Table t(col VARCHAR(200)) Table at(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.ts.")));
+        Assert.assertEquals("[tab , tab}#]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table co(val INTEGER) Table tab(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t")));
         Assert.assertEquals("", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table tab(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.x")));
         Assert.assertEquals("", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table co(val INTEGER) Table tab(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.tab}#")));
     }
@@ -189,8 +193,8 @@ public void testRelationCast()
     @Test
     public void testJoin()
     {
-        Assert.assertEquals("[a::A , >{a::A]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>")));
-        Assert.assertEquals("[t , t}]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.")));
+        Assert.assertEquals("[a::A , >{a::A.]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>")));
+        Assert.assertEquals("[t , t}#]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.")));
         Assert.assertEquals("[JoinKind , JoinKind.]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.t}#, ")));
         Assert.assertEquals("[LEFT , LEFT], [INNER , INNER]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.t}#, JoinKind.")));
         Assert.assertEquals("[col , col], [val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.t}#, JoinKind.INNER, {a,b|$a.")));
@@ -205,8 +209,8 @@ public void testJoin()
     @Test
     public void testAsOfJoin()
     {
-        Assert.assertEquals("[a::A , >{a::A]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>")));
-        Assert.assertEquals("[t , t}]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.")));
+        Assert.assertEquals("[a::A , >{a::A.]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>")));
+        Assert.assertEquals("[t , t}#]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.")));
         Assert.assertEquals("[col , col], [val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t}#, {a,b|$a.")));
         Assert.assertEquals("", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t}#,")));
         Assert.assertEquals("[k , k], [o , o]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t2(k VARCHAR(200), o INT) Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t2}#, {a,b|$a.col == $b.")));

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure

@@ -269,7 +269,12 @@ function meta::relational::tests::pct::process::reprocess(f:Function<Any>[1], db
                                   ],
                                 extraInstanceValueHandlers = 
                                   [
-                                    x:RelationStoreAccessor<Any>[1]|'#>{'+$x.store->elementToPath()+'.'+$x.sourceElement->cast(@Table).name->toOne()+'}#',
+                                    x:RelationStoreAccessor<Any>[1]|
+                                      let table = $x.sourceElement->cast(@Table);  
+                                      let schema = $table.schema;
+                                      let schemaName = if ($schema.name == 'default', |'', |$schema.name + '.');
+                                      '#>{' + $x.store->elementToPath() + '.' + $schemaName + $table.name->toOne() + '}#';
+                                      ,
                                     t:meta::pure::metamodel::relation::TDS<Any>[1]|'#TDS\n'+$t.csv->replace(' ','')+'#'
                                   ]
                             )