Skip to content

Commit

Permalink
Schema support in relation accessor (#3256)
Browse files Browse the repository at this point in the history 
* Support schema on relation accessor

* Update .gs-project.yml

* Fix test case expected error message

* Fix relational PCT setup - respect schema

* Fix parser fixer test
  • Loading branch information
@rafaelbey
rafaelbey authored Nov 19, 2024
1 parent 239d9cc commit 028987c
Show file tree
Hide file tree
Showing 11 changed files with 124 additions and 97 deletions.
2 changes: 1 addition & 1 deletion .gs-project.yml
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
productGuid: "product::315850"
productGuid: "product::776807"
4 changes: 2 additions & 2 deletions ...pl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/parser/ParserFixer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,13 +207,13 @@ else if (buffer)
}
if (content.contains("{") && !content.contains("}"))
{
return value + "}#";
return (value.contains(magicToken) ? value : value + magicToken) + "}#";
}
if (content.contains("{") && content.contains("}"))
{
return value + "#";
}
return value + "{}#";
return value + "{" + magicToken + "}#";
}
return value;

Expand Down
6 changes: 3 additions & 3 deletions ...legend-engine-repl-client/src/test/java/org/finos/legend/engine/repl/TestParserFixer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@ public void testArithmetic()
public void testIsland()
{
Assert.assertEquals("#MaGiCToKeN{}#", fixCode("#"));
Assert.assertEquals("#>{}#", fixCode("#>"));
Assert.assertEquals("#>{}#", fixCode("#>{"));
Assert.assertEquals("#>{MaGiCToKeN}#", fixCode("#>"));
Assert.assertEquals("#>{MaGiCToKeN}#", fixCode("#>{"));
Assert.assertEquals("#>{}#", fixCode("#>{}"));
Assert.assertEquals("#>{a::A.t}#->fil()", fixCode("#>{a::A.t}#->fil"));
Assert.assertEquals("#>{a::A.t}#->x(#>{}#)", fixCode("#>{a::A.t}#->x(#>"));
Assert.assertEquals("#>{a::A.t}#->x(#>{MaGiCToKeN}#)", fixCode("#>{a::A.t}#->x(#>"));
Assert.assertEquals("#>{a::A.t}#->x(#>{a::A.MaGiCToKeN}#)", fixCode("#>{a::A.t}#->x(#>{a::A."));
}

Expand Down
14 changes: 9 additions & 5 deletions legend-engine-config/legend-engine-repl/legend-engine-repl-relational/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,7 @@
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-repl-client</artifactId>
</dependency>

<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m4</artifactId>
</dependency>

<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m3-core</artifactId>
Expand Down Expand Up @@ -151,5 +147,13 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-protocol</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-relationalStore-grammar</artifactId>
</dependency>
</dependencies>
</project>
61 changes: 46 additions & 15 deletions ...va/org/finos/legend/engine/repl/relational/autocomplete/RelationalCompleterExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,22 @@
package org.finos.legend.engine.repl.relational.autocomplete;

import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.api.set.SetIterable;
import org.eclipse.collections.impl.factory.Lists;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.HelperRelationalBuilder;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
import org.finos.legend.engine.protocol.pure.v1.model.SourceInformation;
import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.relation.RelationStoreAccessor;
import org.finos.legend.engine.repl.autocomplete.CompleterExtension;
import org.finos.legend.engine.repl.autocomplete.CompletionItem;
import org.finos.legend.engine.repl.autocomplete.CompletionResult;
import org.finos.legend.engine.repl.autocomplete.parser.ParserFixer;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement;
import org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store;
import org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database;
import org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Schema;
import org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.relation.Table;

public class RelationalCompleterExtension implements CompleterExtension
{
Expand All @@ -33,32 +39,57 @@ public CompletionResult extraClassInstanceProcessor(Object islandExpr, PureModel
{
if (islandExpr instanceof RelationStoreAccessor)
{
MutableList<String> path = Lists.mutable.withAll(((RelationStoreAccessor) islandExpr).path);
String writtenPath = path.makeString("::").replace(ParserFixer.magicToken, "");
MutableList<Store> elements = pureModel.getAllStores().select(c -> nameMatch(c, writtenPath)).toList();
if (elements.size() == 1 &&
writtenPath.startsWith(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(elements.get(0))) &&
!writtenPath.equals(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(elements.get(0)))
)
RelationStoreAccessor relationStoreAccessor = (RelationStoreAccessor) islandExpr;
MutableList<String> path = Lists.adapt(relationStoreAccessor.path);

if (path.anySatisfy(x -> x.isEmpty() || x.contains(ParserFixer.magicToken)))
{
org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database db = (org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database) elements.get(0);
String tableName = writtenPath.replace(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(db), "").replace("::", "");
MutableList<? extends org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.relation.Table> tables = db._schemas().isEmpty() ? Lists.mutable.empty() : db._schemas().getFirst()._tables().toList();
MutableList<? extends org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.relation.Table> foundTables = tables.select(c -> c._name().startsWith(tableName));
if ((foundTables.size() == 1 && foundTables.get(0)._name().equals(path.getLast())))
String writtenPath = path.get(0).replace(ParserFixer.magicToken, "");
MutableList<Store> elements = pureModel.getAllStores().select(c -> nameMatch(c, writtenPath)).toList();

MutableList<CompletionItem> completionItems = Lists.mutable.empty();

if (elements.size() == 1 && path.size() > 1)
{
return new CompletionResult(Lists.mutable.empty());
org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database db = (org.finos.legend.pure.m3.coreinstance.meta.relational.metamodel.Database) elements.get(0);

if (path.size() < 3)
{
String tableOrSchema = path.get(1).replace(ParserFixer.magicToken, "").replace("::", "");
completionItems.addAll(getTableSuggestions(db, "default", tableOrSchema));
completionItems.addAll(getSchemaSuggestions(db, tableOrSchema));
}
else
{
String schema = path.get(1);
String tableName = path.get(2).replace(ParserFixer.magicToken, "").replace("::", "");
completionItems.addAll(getTableSuggestions(db, schema, tableName));
}
}
else
{
return new CompletionResult(foundTables.collect(c -> new CompletionItem(c._name(), c._name() + "}")));
ListIterate.collect(elements, c -> new CompletionItem(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c), ">{" + org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c) + '.'), completionItems);
}

return new CompletionResult(completionItems);
}
return new CompletionResult(ListIterate.collect(elements, c -> new CompletionItem(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c), ">{" + org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c))).toList());
}
return null;
}

private static MutableList<CompletionItem> getSchemaSuggestions(Database db, String schemaName)
{
MutableList<? extends Schema> foundSchemas = db._schemas().select(schema -> !schema._name().equals("default") && schema._name().startsWith(schemaName)).toSortedListBy(Schema::_name);
return foundSchemas.collect(c -> new CompletionItem(c._name(), c._name() + "."));
}

private static MutableList<CompletionItem> getTableSuggestions(Database db, String schemaName, String tableName)
{
SetIterable<Table> tables = HelperRelationalBuilder.getAllTablesInSchema(db, schemaName, SourceInformation.getUnknownSourceInformation());
MutableList<? extends Table> foundTables = tables.select(c -> c._name().startsWith(tableName)).toSortedListBy(Table::_name);
return foundTables.collect(c -> new CompletionItem(c._name(), c._name() + "}#"));
}

private static boolean nameMatch(PackageableElement c, String writtenPath)
{
String path = org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(c);
Expand Down
24 changes: 14 additions & 10 deletions ...gend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,16 @@ public class TestCompleter
@Test
public void testRelationAccessor()
{
Assert.assertEquals("[a::A , >{a::A]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>")));
Assert.assertEquals("[a::other , >{a::other], [a::ABC , >{a::ABC]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a")));
Assert.assertEquals("[a::ABC , >{a::ABC]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A")));
Assert.assertEquals("[a::ABC , >{a::ABC]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC")));
Assert.assertEquals("[t , t}]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.")));
Assert.assertEquals("[tab , tab}]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table co(val INTEGER) Table tab(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t")));
Assert.assertEquals("[a::A , >{a::A.]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>")));
Assert.assertEquals("[a::other , >{a::other.], [a::ABC , >{a::ABC.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a")));
Assert.assertEquals("[a::ABC , >{a::ABC.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A")));
Assert.assertEquals("[a::ABC , >{a::ABC.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC")));
Assert.assertEquals("[t , t}#]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Table t(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.")));
Assert.assertEquals("[at , at}#], [t , t}#], [ts , ts.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Schema ts(Table t(col VARCHAR(200))) Table t(col VARCHAR(200)) Table at(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.")));
Assert.assertEquals("[t , t}#], [ts , ts.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Schema ts(Table t(col VARCHAR(200))) Table t(col VARCHAR(200)) Table at(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.t")));
Assert.assertEquals("[ts , ts.]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Schema ts(Table t(col VARCHAR(200))) Table t(col VARCHAR(200)) Table at(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.ts")));
Assert.assertEquals("[t , t}#]", checkResultNoException(new Completer("###Relational\nDatabase a::ABC(Schema ts(Table t(col VARCHAR(200))) Table t(col VARCHAR(200)) Table at(col VARCHAR(200)))\nDatabase a::other(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::ABC.ts.")));
Assert.assertEquals("[tab , tab}#]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table co(val INTEGER) Table tab(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t")));
Assert.assertEquals("", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table tab(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.x")));
Assert.assertEquals("", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table co(val INTEGER) Table tab(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.tab}#")));
}
Expand Down Expand Up @@ -189,8 +193,8 @@ public void testRelationCast()
@Test
public void testJoin()
{
Assert.assertEquals("[a::A , >{a::A]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>")));
Assert.assertEquals("[t , t}]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.")));
Assert.assertEquals("[a::A , >{a::A.]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>")));
Assert.assertEquals("[t , t}#]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.")));
Assert.assertEquals("[JoinKind , JoinKind.]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.t}#, ")));
Assert.assertEquals("[LEFT , LEFT], [INNER , INNER]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.t}#, JoinKind.")));
Assert.assertEquals("[col , col], [val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.t}#, JoinKind.INNER, {a,b|$a.")));
Expand All @@ -205,8 +209,8 @@ public void testJoin()
@Test
public void testAsOfJoin()
{
Assert.assertEquals("[a::A , >{a::A]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>")));
Assert.assertEquals("[t , t}]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.")));
Assert.assertEquals("[a::A , >{a::A.]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>")));
Assert.assertEquals("[t , t}#]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.")));
Assert.assertEquals("[col , col], [val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t}#, {a,b|$a.")));
Assert.assertEquals("", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t}#,")));
Assert.assertEquals("[k , k], [o , o]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t2(k VARCHAR(200), o INT) Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t2}#, {a,b|$a.col == $b.")));
Expand Down
7 changes: 6 additions & 1 deletion ...tionalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,12 @@ function meta::relational::tests::pct::process::reprocess(f:Function<Any>[1], db
],
extraInstanceValueHandlers =
[
x:RelationStoreAccessor<Any>[1]|'#>{'+$x.store->elementToPath()+'.'+$x.sourceElement->cast(@Table).name->toOne()+'}#',
x:RelationStoreAccessor<Any>[1]|
let table = $x.sourceElement->cast(@Table);
let schema = $table.schema;
let schemaName = if ($schema.name == 'default', |'', |$schema.name + '.');
'#>{' + $x.store->elementToPath() + '.' + $schemaName + $table.name->toOne() + '}#';
,
t:meta::pure::metamodel::relation::TDS<Any>[1]|'#TDS\n'+$t.csv->replace(' ','')+'#'
]
)
Expand Down
Loading

0 comments on commit 028987c

Please sign in to comment.