From feae92b095746557ef29c70a20cea71a9127d6a3 Mon Sep 17 00:00:00 2001 From: Rafael Bey <24432403+rafaelbey@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:28:59 -0500 Subject: [PATCH] Reuse schema exploration on REPL to generate database to support same Db to model translation (#3299) --- .../repl/dataCube/commands/DataCubeCache.java | 25 +++--- .../repl/dataCube/commands/DataCubeTable.java | 18 ++-- .../commands/DataCubeWalkthrough.java | 34 +++---- .../dataCube/shared/DataCubeSampleData.java | 42 +++++---- .../repl/relational/commands/Cache.java | 25 +++--- .../engine/repl/relational/commands/DB.java | 22 +++-- .../engine/repl/relational/commands/Drop.java | 18 ++-- .../engine/repl/relational/commands/Load.java | 20 +++-- .../local/LocalConnectionManagement.java | 31 +++---- .../engine/repl/relational/schema/Column.java | 27 ------ .../relational/schema/MetadataReader.java | 90 ------------------- .../engine/repl/relational/schema/Table.java | 31 ------- .../relational/shared/ConnectionHelper.java | 56 ++++++++++-- .../finos/legend/engine/server/Server.java | 2 +- .../pom.xml | 20 ----- .../api/schema/SchemaExplorationApi.java | 15 ++-- .../exploration}/SchemaExportation.java | 72 +++++++-------- .../model/DatabaseBuilderConfig.java | 5 +- .../model/DatabaseBuilderInput.java | 5 +- .../exploration}/model/DatabasePattern.java | 5 +- .../exploration}/model/TargetDatabase.java | 16 +++- .../exploration}/TestSchemaExploration.java | 14 +-- 22 files changed, 236 insertions(+), 357 deletions(-) delete mode 100644 legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/Column.java delete mode 100644 legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/MetadataReader.java delete mode 100644 legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/Table.java rename legend-engine-xts-relationalStore/{legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema => legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration}/SchemaExportation.java (89%) rename legend-engine-xts-relationalStore/{legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema => legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration}/model/DatabaseBuilderConfig.java (94%) rename legend-engine-xts-relationalStore/{legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema => legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration}/model/DatabaseBuilderInput.java (94%) rename legend-engine-xts-relationalStore/{legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema => legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration}/model/DatabasePattern.java (97%) rename legend-engine-xts-relationalStore/{legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema => legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration}/model/TargetDatabase.java (78%) rename legend-engine-xts-relationalStore/{legend-engine-xt-relationalStore-connection-http-api/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema => legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/exploration}/TestSchemaExploration.java (98%) diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeCache.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeCache.java index ee4443c49c6..bbff13d5a32 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeCache.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeCache.java @@ -14,6 +14,11 @@ package org.finos.legend.engine.repl.dataCube.commands; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.stream.Collectors; import org.eclipse.collections.api.list.MutableList; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.plan.execution.result.Result; @@ -26,25 +31,19 @@ import org.finos.legend.engine.plan.execution.stores.relational.serialization.RelationalResultToCSVSerializerWithTransformersApplied; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Table; import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.core.Command; import org.finos.legend.engine.repl.dataCube.server.REPLServer; import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; +import static org.finos.legend.engine.repl.shared.ExecutionHelper.REPL_RUN_FUNCTION_SIGNATURE; +import static org.finos.legend.engine.repl.shared.ExecutionHelper.executeCode; import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; import org.jline.reader.Candidate; import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; - -import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; -import static org.finos.legend.engine.repl.shared.ExecutionHelper.REPL_RUN_FUNCTION_SIGNATURE; -import static org.finos.legend.engine.repl.shared.ExecutionHelper.executeCode; - public class DataCubeCache implements Command { private final DataCube parentCommand; @@ -94,7 +93,7 @@ public boolean process(String line) throws Exception this.client.printError("Failed to retrieve the last command"); return true; } - DatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), DataCube.getLocalConnectionPath()); + RelationalDatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), DataCube.getLocalConnectionPath()); try { @@ -117,9 +116,11 @@ public boolean process(String line) throws Exception throw new RuntimeException(e); } + List tables = ConnectionHelper.getTables(databaseConnection, client.getPlanExecutor()).collect(Collectors.toList()); + try (Connection connection = ConnectionHelper.getConnection(databaseConnection, client.getPlanExecutor())) { - String tableName = specifiedTableName != null ? specifiedTableName : "test" + (getTables(connection).size() + 1); + String tableName = specifiedTableName != null ? specifiedTableName : "test" + (tables.size() + 1); try (Statement statement = connection.createStatement()) { statement.executeUpdate(DatabaseManager.fromString(databaseConnection.type.name()).relationalDatabaseSupport().load(tableName, tempFile.getTemporaryPathForFile(), relationalResultColumns)); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeTable.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeTable.java index ae654bf131d..842635d5422 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeTable.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeTable.java @@ -14,6 +14,7 @@ package org.finos.legend.engine.repl.dataCube.commands; +import java.util.stream.Collectors; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.factory.Lists; import org.eclipse.collections.impl.utility.ListIterate; @@ -21,19 +22,17 @@ import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Database; import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.core.Command; import org.finos.legend.engine.repl.dataCube.server.REPLServer; import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; +import static org.finos.legend.engine.repl.shared.ExecutionHelper.REPL_RUN_FUNCTION_SIGNATURE; import org.jline.reader.Candidate; import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; -import java.sql.Connection; - -import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; -import static org.finos.legend.engine.repl.shared.ExecutionHelper.REPL_RUN_FUNCTION_SIGNATURE; - public class DataCubeTable implements Command { private final DataCube parentCommand; @@ -101,14 +100,7 @@ public MutableList complete(String inScope, LineReader lineReader, Pa .select(c -> PureGrammarComposerUtility.convertPath(c.getPath()).equals(DataCube.getLocalConnectionPath())); if (!foundConnections.isEmpty() && foundConnections.getFirst().connectionValue instanceof DatabaseConnection) { - try (Connection connection = ConnectionHelper.getConnection((DatabaseConnection) foundConnections.getFirst().connectionValue, client.getPlanExecutor())) - { - return getTables(connection).select(c -> c.name.startsWith(start)).collect(c -> c.name).collect(Candidate::new); - } - catch (Exception e) - { - // do nothing - } + return ConnectionHelper.getTables((RelationalDatabaseConnection) foundConnections.getFirst().connectionValue, client.getPlanExecutor()).filter(c -> c.name.startsWith(start)).map(c -> c.name).map(Candidate::new).collect(Collectors.toCollection(Lists.mutable::empty)); } return null; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java index e6335c9b218..f66faf7e376 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java @@ -14,37 +14,39 @@ package org.finos.legend.engine.repl.dataCube.commands; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Statement; +import java.util.Objects; +import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.eclipse.collections.api.block.function.Function0; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.MutableList; import org.finos.legend.engine.plan.execution.stores.relational.connection.driver.DatabaseManager; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Table; import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.core.Command; import org.finos.legend.engine.repl.core.commands.Execute; import org.finos.legend.engine.repl.dataCube.server.REPLServer; import org.finos.legend.engine.repl.dataCube.shared.DataCubeSampleData; import org.finos.legend.engine.repl.relational.RelationalReplExtension; -import org.finos.legend.engine.repl.relational.schema.Table; import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; import org.finos.legend.engine.repl.shared.DocumentationHelper; +import static org.finos.legend.engine.repl.shared.REPLHelper.ansiDim; +import static org.finos.legend.engine.repl.shared.REPLHelper.ansiGreen; +import static org.finos.legend.engine.repl.shared.REPLHelper.ansiYellow; +import static org.finos.legend.engine.repl.shared.REPLHelper.getLineWidth; +import static org.finos.legend.engine.repl.shared.REPLHelper.wrap; import org.jline.reader.Candidate; import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.Statement; -import java.util.Objects; - -import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; -import static org.finos.legend.engine.repl.shared.REPLHelper.*; - public class DataCubeWalkthrough implements Command { private final DataCube parentCommand; @@ -155,13 +157,15 @@ public static class DataCubeWalkthrough1 extends DocumentationHelper.Walkthrough @Override public void beforeStep() { - DatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), DataCube.getLocalConnectionPath()); + RelationalDatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), DataCube.getLocalConnectionPath()); + + MutableList
tables = ConnectionHelper.getTables(databaseConnection, this.client.getPlanExecutor()).collect(Collectors.toCollection(Lists.mutable::empty)); + try ( InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream("org/finos/legend/engine/repl/dataCube/walkthrough/sport-data.csv"); Connection connection = ConnectionHelper.getConnection(databaseConnection, client.getPlanExecutor()); Statement statement = connection.createStatement()) { - MutableList
tables = getTables(connection); if (tables.anySatisfy(table -> table.name.equals(DataCubeSampleData.SPORT.tableName))) { this.tableName = DataCubeSampleData.SPORT.tableName; diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/shared/DataCubeSampleData.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/shared/DataCubeSampleData.java index 56a47a65ff9..4a0afcc8e09 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/shared/DataCubeSampleData.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/shared/DataCubeSampleData.java @@ -14,16 +14,6 @@ package org.finos.legend.engine.repl.dataCube.shared; -import org.apache.commons.io.IOUtils; -import org.eclipse.collections.api.list.MutableList; -import org.eclipse.collections.impl.factory.Lists; -import org.finos.legend.engine.plan.execution.stores.relational.connection.driver.DatabaseManager; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; -import org.finos.legend.engine.repl.client.Client; -import org.finos.legend.engine.repl.dataCube.commands.DataCube; -import org.finos.legend.engine.repl.relational.schema.Table; -import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; - import java.io.FileOutputStream; import java.io.InputStream; import java.nio.file.Files; @@ -32,8 +22,16 @@ import java.sql.Statement; import java.util.Arrays; import java.util.Objects; - -import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; +import java.util.stream.Collectors; +import org.apache.commons.io.IOUtils; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.plan.execution.stores.relational.connection.driver.DatabaseManager; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Table; +import org.finos.legend.engine.repl.client.Client; +import org.finos.legend.engine.repl.dataCube.commands.DataCube; +import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; public class DataCubeSampleData { @@ -55,13 +53,13 @@ public DataCubeSampleData(String name, String tableName, String csvFilePath, Mut public void load(Client client) { - DatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(client.getModelState().parse(), DataCube.getLocalConnectionPath()); + RelationalDatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(client.getModelState().parse(), DataCube.getLocalConnectionPath()); + MutableList
tables = ConnectionHelper.getTables(databaseConnection, client.getPlanExecutor()).collect(Collectors.toCollection(Lists.mutable::empty)); try ( InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(this.csvFilePath); Connection connection = ConnectionHelper.getConnection(databaseConnection, client.getPlanExecutor()); Statement statement = connection.createStatement()) { - MutableList
tables = getTables(connection); if (tables.anySatisfy(t -> t.name.equals(this.tableName))) { statement.executeUpdate(DatabaseManager.fromString(databaseConnection.type.name()).relationalDatabaseSupport().dropTable(tableName)); @@ -71,18 +69,18 @@ public void load(Client client) IOUtils.copy(Objects.requireNonNull(inputStream, "Can't extract sample data '" + this.name + "' from " + this.csvFilePath), outputStream); outputStream.close(); // explicitly close output stream to let database access this file, else this would throw `java.sql.SQLException: IO Error: File is already open in` error on Windows statement.executeUpdate(DatabaseManager.fromString(databaseConnection.type.name()).relationalDatabaseSupport().load(tableName, tempFile.toString())); - - // post check - tables = getTables(connection); - Table table = tables.detect(t -> t.name.equals(this.tableName)); - if (!Arrays.equals(table.columns.collect(column -> column.name).toArray(), this.expectedColumns.toArray())) - { - throw new RuntimeException("Sample data '" + this.name + "' does not have the expected columns " + this.expectedColumns.makeString("(", ",", ")") + " (got: " + table.columns.collect(column -> column.name).makeString(",") + ")"); - } } catch (Exception e) { throw new RuntimeException(e); } + + // post check + tables = ConnectionHelper.getTables(databaseConnection, client.getPlanExecutor()).collect(Collectors.toCollection(Lists.mutable::empty));; + Table table = tables.detect(t -> t.name.equals(this.tableName)); + if (!Arrays.equals(table.columns.stream().map(column -> column.name).toArray(), this.expectedColumns.toArray())) + { + throw new RuntimeException("Sample data '" + this.name + "' does not have the expected columns " + this.expectedColumns.makeString("(", ",", ")") + " (got: " + table.columns.stream().map(column -> column.name).collect(Collectors.joining(",")) + ")"); + } } } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java index 7758366d26a..695084dd6e7 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java @@ -14,6 +14,11 @@ package org.finos.legend.engine.repl.relational.commands; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.stream.Collectors; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.factory.Lists; import org.eclipse.collections.impl.utility.ListIterate; @@ -30,24 +35,18 @@ import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Table; import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.core.Command; import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; +import static org.finos.legend.engine.repl.shared.ExecutionHelper.executeCode; +import static org.finos.legend.engine.repl.shared.ExecutionHelper.printExecutionTime; import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; import org.jline.reader.Candidate; import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; - -import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; -import static org.finos.legend.engine.repl.shared.ExecutionHelper.executeCode; -import static org.finos.legend.engine.repl.shared.ExecutionHelper.printExecutionTime; - public class Cache implements Command { private final Client client; @@ -88,7 +87,7 @@ public boolean process(String line) throws Exception this.client.printError("Failed to retrieve the last command"); return true; } - DatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), connectionPath); + RelationalDatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), connectionPath); try { @@ -112,9 +111,11 @@ public boolean process(String line) throws Exception throw new RuntimeException(e); } + List
tables = ConnectionHelper.getTables(databaseConnection, client.getPlanExecutor()).collect(Collectors.toList()); + try (Connection connection = ConnectionHelper.getConnection(databaseConnection, client.getPlanExecutor())) { - String tableName = specifiedTableName != null ? specifiedTableName : "test" + (getTables(connection).size() + 1); + String tableName = specifiedTableName != null ? specifiedTableName : "test" + (tables.size() + 1); try (Statement statement = connection.createStatement()) { statement.executeUpdate(DatabaseManager.fromString(databaseConnection.type.name()).relationalDatabaseSupport().load(tableName, tempFile.getTemporaryPathForFile(), relationalResultColumns)); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/DB.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/DB.java index 1a568ecad18..9adbaf4073c 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/DB.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/DB.java @@ -17,10 +17,14 @@ import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.factory.Lists; import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.pure.grammar.from.RelationalGrammarParserExtension; +import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposer; +import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerContext; import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerUtility; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Database; import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.core.Command; import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; @@ -28,10 +32,6 @@ import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; -import java.sql.Connection; - -import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; - public class DB implements Command { private final Client client; @@ -65,14 +65,12 @@ public boolean process(String line) throws Exception } PureModelContextData d = this.client.getModelState().parse(); - DatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(d, tokens[1]); + RelationalDatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(d, tokens[1]); - try (Connection connection = ConnectionHelper.getConnection(databaseConnection, client.getPlanExecutor())) - { - this.client.println( - getTables(connection).collect(c -> c.schema + "." + c.name + "(" + c.columns.collect(col -> PureGrammarComposerUtility.convertIdentifier(col.name, true) + " " + col.type).makeString(", ") + ")").makeString("\n") - ); - } + Database database = ConnectionHelper.getDatabase(databaseConnection, "local", "DB", this.client.getPlanExecutor()); + this.client.println( + PureGrammarComposer.newInstance(PureGrammarComposerContext.Builder.newInstance().build()).render(database, RelationalGrammarParserExtension.NAME) + ); return true; } return false; diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Drop.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Drop.java index 26b2352e2f6..ab8b26bf288 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Drop.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Drop.java @@ -14,6 +14,9 @@ package org.finos.legend.engine.repl.relational.commands; +import java.sql.Connection; +import java.sql.Statement; +import java.util.stream.Collectors; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.factory.Lists; import org.eclipse.collections.impl.utility.ListIterate; @@ -22,6 +25,7 @@ import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.core.Command; import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; @@ -29,11 +33,6 @@ import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; -import java.sql.Connection; -import java.sql.Statement; - -import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; - public class Drop implements Command { private final Client client; @@ -107,14 +106,7 @@ public MutableList complete(String inScope, LineReader lineReader, Pa .select(c -> PureGrammarComposerUtility.convertPath(c.getPath()).equals(connectionPath)); if (!foundConnections.isEmpty() && foundConnections.getFirst().connectionValue instanceof DatabaseConnection) { - try (Connection connection = ConnectionHelper.getConnection((DatabaseConnection) foundConnections.getFirst().connectionValue, client.getPlanExecutor())) - { - return getTables(connection).select(c -> c.name.startsWith(start)).collect(c -> c.name).collect(Candidate::new); - } - catch (Exception e) - { - // do nothing - } + return ConnectionHelper.getTables((RelationalDatabaseConnection) foundConnections.getFirst().connectionValue, client.getPlanExecutor()).filter(c -> c.name.startsWith(start)).map(c -> c.name).map(Candidate::new).collect(Collectors.toCollection(Lists.mutable::empty)); } return null; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Load.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Load.java index 13a4a6138bf..8b8c1a00f97 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Load.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Load.java @@ -14,6 +14,11 @@ package org.finos.legend.engine.repl.relational.commands; +import java.io.File; +import java.sql.Connection; +import java.sql.Statement; +import java.util.List; +import java.util.stream.Collectors; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.factory.Lists; import org.eclipse.collections.impl.utility.ListIterate; @@ -21,7 +26,8 @@ import org.finos.legend.engine.plan.execution.stores.relational.connection.driver.DatabaseManager; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Table; import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.client.jline3.JLine3Parser; import org.finos.legend.engine.repl.core.Command; @@ -31,12 +37,6 @@ import org.jline.reader.LineReader; import org.jline.reader.ParsedLine; -import java.io.File; -import java.sql.Connection; -import java.sql.Statement; - -import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; - public class Load implements Command { private final Client client; @@ -70,11 +70,13 @@ public boolean process(String line) throws Exception throw new RuntimeException("Command should be used as '" + this.documentation() + "'"); } - DatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), tokens[2]); + RelationalDatabaseConnection databaseConnection = ConnectionHelper.getDatabaseConnection(this.client.getModelState().parse(), tokens[2]); + + List
tables = ConnectionHelper.getTables(databaseConnection, client.getPlanExecutor()).collect(Collectors.toList()); try (Connection connection = ConnectionHelper.getConnection(databaseConnection, client.getPlanExecutor())) { - String tableName = tokens.length == 4 ? tokens[3] : ("test" + (getTables(connection).size() + 1)); + String tableName = tokens.length == 4 ? tokens[3] : ("test" + (tables.size() + 1)); try (Statement statement = connection.createStatement()) { diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/local/LocalConnectionManagement.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/local/LocalConnectionManagement.java index e1f91c61b50..79a132d1e78 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/local/LocalConnectionManagement.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/local/LocalConnectionManagement.java @@ -16,17 +16,16 @@ import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.language.pure.grammar.from.RelationalGrammarParserExtension; +import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposer; +import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerContext; import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerUtility; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Database; import org.finos.legend.engine.repl.client.Client; import org.finos.legend.engine.repl.relational.shared.ConnectionHelper; -import java.sql.Connection; -import java.sql.SQLException; - -import static org.finos.legend.engine.repl.relational.schema.MetadataReader.getTables; - public class LocalConnectionManagement { private Client client; @@ -66,21 +65,15 @@ public MutableList generateDynamicContent(String code) return this.connectionsForDynamicDBs.flatCollect(conn -> { String connectionPath = "local::" + conn + "Connection"; - DatabaseConnection db = ConnectionHelper.getDatabaseConnection(pureModelContextData, connectionPath); + RelationalDatabaseConnection db = ConnectionHelper.getDatabaseConnection(pureModelContextData, connectionPath); MutableList res = Lists.mutable.empty(); - try (Connection connection = ConnectionHelper.getConnection(db, client.getPlanExecutor())) - { - res.add("###Relational\n" + - "Database local::" + conn + "Database" + - "(" + - getTables(connection).collect(table -> "Table " + table.name + "(" + table.columns.collect(c -> PureGrammarComposerUtility.convertIdentifier(c.name, true) + " " + c.type).makeString(",") + ")").makeString("\n") + - ")\n"); - } - catch (SQLException e) - { - throw new RuntimeException(e); - } + Database database = ConnectionHelper.getDatabase(db, "local", conn + "Database", this.client.getPlanExecutor()); + + String dbElement = "###Relational\n" + PureGrammarComposer.newInstance(PureGrammarComposerContext.Builder.newInstance().build()) + .render(database, RelationalGrammarParserExtension.NAME); + + res.add(dbElement); res.add("###Runtime\n" + "Runtime local::" + conn + "Runtime\n" + diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/Column.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/Column.java deleted file mode 100644 index 585255f2d60..00000000000 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/Column.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.engine.repl.relational.schema; - -public class Column -{ - public String name; - public String type; - - public Column(String name, String type) - { - this.name = name; - this.type = type; - } -} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/MetadataReader.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/MetadataReader.java deleted file mode 100644 index 2e64b61c98a..00000000000 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/MetadataReader.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.engine.repl.relational.schema; - -import org.eclipse.collections.api.factory.Lists; -import org.eclipse.collections.api.list.MutableList; - -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.JDBCType; -import java.sql.ResultSet; - -public class MetadataReader -{ - public static MutableList
getTables(Connection connection) - { - try - { - MutableList
tables = Lists.mutable.empty(); - DatabaseMetaData metaData = connection.getMetaData(); - - try (ResultSet res = metaData.getTables(null, null, null, null)) - { - while (res.next()) - { - if (!"INFORMATION_SCHEMA".equals(res.getString("TABLE_SCHEM"))) - { - MutableList cols = Lists.mutable.empty(); - try (ResultSet columns = metaData.getColumns(null, null, res.getString("TABLE_NAME"), null)) - { - while (columns.next()) - { - String columnName = columns.getString("COLUMN_NAME"); - cols.add(new Column(columnName, getType(columns))); - } - } - tables.add(new Table(res.getString("TABLE_SCHEM"), res.getString("TABLE_NAME"), cols)); - } - } - } - return tables; - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - public static String getType(ResultSet colInfo) throws Exception - { - int datatype = colInfo.getInt("DATA_TYPE"); - int size = colInfo.getInt("COLUMN_SIZE"); - String name = JDBCType.valueOf(datatype).getName(); - String newType; - switch (name) - { - case "BOOLEAN": - newType = "Bit"; - break; - case "VARCHAR": - newType = "VARCHAR(" + size + ")"; - break; - default: - newType = name; - } - return newType; - } - - -// public static void printSchema(ResultSet res) throws Exception -// { -// ResultSetMetaData m = res.getMetaData(); -// for (int i = 1; i <= m.getColumnCount(); i++) -// { -// System.out.println("-" + m.getColumnName(i)); -// } -// } -} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/Table.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/Table.java deleted file mode 100644 index 4dc4f8d822d..00000000000 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/schema/Table.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.engine.repl.relational.schema; - -import org.eclipse.collections.api.list.MutableList; - -public class Table -{ - public String schema; - public String name; - public MutableList columns; - - public Table(String schema, String name, MutableList columns) - { - this.schema = schema; - this.name = name; - this.columns = columns; - } -} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/shared/ConnectionHelper.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/shared/ConnectionHelper.java index 8a1838338c7..dd445c89561 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/shared/ConnectionHelper.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/shared/ConnectionHelper.java @@ -14,33 +14,79 @@ package org.finos.legend.engine.repl.relational.shared; +import java.sql.SQLException; +import java.util.stream.Stream; +import org.eclipse.collections.impl.factory.Lists; import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.engine.plan.execution.PlanExecutor; import org.finos.legend.engine.plan.execution.stores.StoreType; import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.SchemaExportation; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.model.DatabaseBuilderInput; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.model.DatabasePattern; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.model.TargetDatabase; import org.finos.legend.engine.plan.execution.stores.relational.plugin.RelationalStoreExecutor; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Database; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Table; import org.finos.legend.engine.shared.core.identity.Identity; public class ConnectionHelper { - public static java.sql.Connection getConnection(DatabaseConnection connection, PlanExecutor planExecutor) + private static ConnectionManagerSelector getConnectionManagerSelector(PlanExecutor planExecutor) { RelationalStoreExecutor r = (RelationalStoreExecutor) planExecutor.getExecutorsOfType(StoreType.Relational).getFirst(); - ConnectionManagerSelector connectionManager = r.getStoreState().getRelationalExecutor().getConnectionManager(); + return r.getStoreState().getRelationalExecutor().getConnectionManager(); + } + + public static java.sql.Connection getConnection(DatabaseConnection connection, PlanExecutor planExecutor) + { + ConnectionManagerSelector connectionManager = getConnectionManagerSelector(planExecutor); return connection == null ? connectionManager.getTestDatabaseConnection() : connectionManager.getDatabaseConnection(new Identity("X"), connection); } - public static DatabaseConnection getDatabaseConnection(PureModelContextData d, String path) + public static Database getDatabase(RelationalDatabaseConnection connection, String _package, String name, PlanExecutor planExecutor) + { + try + { + SchemaExportation schemaExportation = SchemaExportation.newBuilder(ConnectionHelper.getConnectionManagerSelector(planExecutor)); + DatabaseBuilderInput databaseBuilderInput = new DatabaseBuilderInput(); + databaseBuilderInput.connection = connection; + databaseBuilderInput.targetDatabase = new TargetDatabase(_package, name); + databaseBuilderInput.config.patterns = Lists.fixedSize.of(new DatabasePattern(null, null)); + databaseBuilderInput.config.enrichTables = true; + databaseBuilderInput.config.enrichColumns = true; + databaseBuilderInput.config.enrichPrimaryKeys = true; + Database database = schemaExportation.build(databaseBuilderInput, Identity.getAnonymousIdentity()); + if (connection.databaseType == DatabaseType.DuckDB) + { + database.schemas.stream().filter(x -> x.name.equals("main")).forEach(x -> x.name = "default"); + } + return database; + } + catch (SQLException e) + { + throw new RuntimeException(e); + } + } + + public static Stream
getTables(RelationalDatabaseConnection connection, PlanExecutor planExecutor) + { + Database database = ConnectionHelper.getDatabase(connection, "", "", planExecutor); + return database.schemas.stream().flatMap(x -> x.tables.stream()); + } + + public static RelationalDatabaseConnection getDatabaseConnection(PureModelContextData d, String path) { PackageableConnection packageableConnection = ListIterate.select(d.getElementsOfType(PackageableConnection.class), c -> c.getPath().equals(path)).getFirst(); if (packageableConnection == null) { throw new RuntimeException("Error, the connection '" + path + "' can't be found!"); } - DatabaseConnection databaseConnection = (DatabaseConnection) packageableConnection.connectionValue; - return databaseConnection; + return (RelationalDatabaseConnection) packageableConnection.connectionValue; } } diff --git a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java index b948dc633e7..19945f917c9 100644 --- a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java +++ b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java @@ -354,7 +354,7 @@ public void run(T serverConfiguration, Environment environment) environment.jersey().register(new TransformRelationalOperationElementJsonToGrammar()); // Relational - environment.jersey().register(new SchemaExplorationApi(modelManager, relationalStoreExecutor)); + environment.jersey().register(new SchemaExplorationApi(relationalStoreExecutor)); environment.jersey().register(new RelationalElementAPI(serverConfiguration.deployment.mode, relationalStoreExecutor)); // Compilation diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/pom.xml index 4ca263c5b7b..f214ed5f875 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/pom.xml @@ -55,36 +55,16 @@ org.pac4j.jax-rscore - - org.apache.commons - commons-lang3 - org.pac4j pac4j-core - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - org.eclipse.collections eclipse-collections-api - - org.eclipse.collections - eclipse-collections - diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/SchemaExplorationApi.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/SchemaExplorationApi.java index fb2260b3184..6b4e3531d37 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/SchemaExplorationApi.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/SchemaExplorationApi.java @@ -19,9 +19,10 @@ import io.swagger.annotations.ApiParam; import org.eclipse.collections.api.list.MutableList; import org.finos.legend.engine.language.pure.modelManager.ModelManager; -import org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model.DatabaseBuilderInput; import org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model.RawSQLExecuteInput; import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.SchemaExportation; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.model.DatabaseBuilderInput; import org.finos.legend.engine.plan.execution.stores.relational.plugin.RelationalStoreExecutor; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Database; @@ -46,13 +47,17 @@ @Path("pure/v1/utilities/database") public class SchemaExplorationApi { - private final ModelManager modelManager; private final ConnectionManagerSelector connectionManager; + @Deprecated public SchemaExplorationApi(ModelManager modelManager, RelationalStoreExecutor relationalStoreExecutor) { - this.modelManager = modelManager; + this(relationalStoreExecutor); + } + + public SchemaExplorationApi(RelationalStoreExecutor relationalStoreExecutor) + { this.connectionManager = relationalStoreExecutor.getStoreState().getRelationalExecutor().getConnectionManager(); } @@ -67,8 +72,8 @@ public Response buildDatabase(DatabaseBuilderInput databaseBuilderInput, @ApiPar Identity identity = Identity.makeIdentity(profiles); try { - SchemaExportation databaseBuilder = SchemaExportation.newBuilder(databaseBuilderInput); - Database database = databaseBuilder.build(this.connectionManager, identity); + SchemaExportation databaseBuilder = SchemaExportation.newBuilder(this.connectionManager); + Database database = databaseBuilder.build(databaseBuilderInput, identity); PureModelContextData graph = PureModelContextData.newBuilder().withElement(database).build(); return Response.ok(graph, MediaType.APPLICATION_JSON_TYPE).build(); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/SchemaExportation.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/SchemaExportation.java similarity index 89% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/SchemaExportation.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/SchemaExportation.java index 117cd512a01..c489bdce5e7 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/SchemaExportation.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/SchemaExportation.java @@ -1,4 +1,4 @@ -// Copyright 2021 Goldman Sachs +// Copyright 2024 Goldman Sachs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,14 +11,24 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// -package org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema; +package org.finos.legend.engine.plan.execution.stores.relational.exploration; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Predicate; import org.apache.commons.lang3.StringUtils; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Maps; @@ -27,10 +37,10 @@ import org.eclipse.collections.impl.map.mutable.UnifiedMap; import org.eclipse.collections.impl.tuple.Tuples; import org.eclipse.collections.impl.utility.ListIterate; -import org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model.DatabaseBuilderConfig; -import org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model.DatabaseBuilderInput; -import org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model.DatabasePattern; import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.model.DatabaseBuilderConfig; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.model.DatabaseBuilderInput; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.model.DatabasePattern; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Column; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Database; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Schema; @@ -40,19 +50,9 @@ import org.finos.legend.engine.shared.core.ObjectMapperFactory; import org.finos.legend.engine.shared.core.identity.Identity; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Types; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.function.Predicate; - public class SchemaExportation { - private final DatabaseBuilderInput databaseBuilderInput; + private final ConnectionManagerSelector connectionManager; private static final String ESCAPE_CHARS = " :"; private static final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @@ -93,21 +93,21 @@ public class SchemaExportation Tuples.pair(Types.CHAR, size -> createDataType("Char", size, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.datatype.Char.class)) ); - private static final String[] TABLES_TYPES = new String[]{"TABLE", "VIEW"}; + private static final String[] TABLES_TYPES = new String[] {"TABLE", "VIEW", "BASE TABLE"}; private static final String DEFAULT_SCHEMA = "default"; - SchemaExportation(DatabaseBuilderInput storeBuilderInput) + SchemaExportation(ConnectionManagerSelector connectionManager) { - this.databaseBuilderInput = storeBuilderInput; + this.connectionManager = connectionManager; } - public static SchemaExportation newBuilder(DatabaseBuilderInput storeBuilderInput) + public static SchemaExportation newBuilder(ConnectionManagerSelector connectionManager) { - return new SchemaExportation(storeBuilderInput); + return new SchemaExportation(connectionManager); } - public Database build(ConnectionManagerSelector connectionManager, Identity identity) throws SQLException + public Database build(DatabaseBuilderInput databaseBuilderInput, Identity identity) throws SQLException { List> tableNameFilters = FastList.newList(); @@ -118,20 +118,20 @@ public Database build(ConnectionManagerSelector connectionManager, Identity iden try (Connection connection = connectionManager.getDatabaseConnection(identity, databaseBuilderInput.connection)) { DatabaseMetaData metadata = connection.getMetaData(); - DatabaseBuilderConfig config = this.databaseBuilderInput.config; + DatabaseBuilderConfig config = databaseBuilderInput.config; Database database = new Database(); - database._package = this.databaseBuilderInput.targetDatabase._package; - database.name = this.databaseBuilderInput.targetDatabase.name; + database._package = databaseBuilderInput.targetDatabase._package; + database.name = databaseBuilderInput.targetDatabase.name; if (config.patterns == null || config.patterns.isEmpty()) { return database; } - this.preProcessInput(this.databaseBuilderInput); + this.preProcessInput(databaseBuilderInput); database.schemas = FastList.newList(); for (DatabasePattern pattern : config.patterns) { - buildDatabaseSchemas(database, metadata, pattern, tableNameFilters); + buildDatabaseSchemas(databaseBuilderInput, database, metadata, pattern, tableNameFilters); } schemaNameMappers.add(SchemaExportation::escapeString); @@ -161,9 +161,9 @@ private String applyNameMapper(List> mappers, String na return mappers.stream().reduce(Function::andThen).orElse(Function.identity()).apply(name); } - private void buildDatabaseSchemas(Database db, DatabaseMetaData metadata, DatabasePattern pattern, List> tableNameFilters) throws SQLException + private void buildDatabaseSchemas(DatabaseBuilderInput databaseBuilderInput, Database db, DatabaseMetaData metadata, DatabasePattern pattern, List> tableNameFilters) throws SQLException { - DatabaseBuilderConfig config = this.databaseBuilderInput.config; + DatabaseBuilderConfig config = databaseBuilderInput.config; // build schemas Map> tablesBySchema = this.buildSchemasAndCollectTables(db, metadata, pattern, tableNameFilters); @@ -185,7 +185,7 @@ private void buildDatabaseSchemas(Database db, DatabaseMetaData metadata, Databa Schema schema = getOrCreateAndAddSchema(db, catalogTablesSchema); for (CatalogTable t : catalogTables) { - buildSchemaTable(t.getCatalog(), schema, t.getTable(), metadata); + buildSchemaTable(databaseBuilderInput, t.getCatalog(), schema, t.getTable(), metadata); } } } @@ -228,7 +228,7 @@ private Map> buildSchemasAndCollectTables(Database db private String escapePattern(String pattern, boolean doEscape, String escapeStringCharacter) { - return pattern != null && doEscape + return pattern != null && doEscape && escapeStringCharacter != null ? pattern.replace("_", escapeStringCharacter + "_") .replace("%", escapeStringCharacter + "%") : pattern; @@ -252,24 +252,24 @@ private String correctCasePattern(String pattern, DatabaseMetaData metaData) thr return pattern; } - private void buildSchemaTable(String catalog, Schema schema, String tableName, DatabaseMetaData metaData) throws SQLException + private void buildSchemaTable(DatabaseBuilderInput databaseBuilderInput, String catalog, Schema schema, String tableName, DatabaseMetaData metaData) throws SQLException { if (ListIterate.noneSatisfy(schema.tables, t -> t.name.equals(tableName))) { Table table = new Table(); table.name = tableName; - DatabaseBuilderConfig config = this.databaseBuilderInput.config; + DatabaseBuilderConfig config = databaseBuilderInput.config; if (config.enrichColumns) { - buildTableColumns(catalog, schema, table, metaData); + buildTableColumns(databaseBuilderInput, catalog, schema, table, metaData); } schema.tables.add(table); } } - private void buildTableColumns(String catalog, Schema schema, Table table, DatabaseMetaData metaData) throws SQLException + private void buildTableColumns(DatabaseBuilderInput databaseBuilderInput, String catalog, Schema schema, Table table, DatabaseMetaData metaData) throws SQLException { - if (this.databaseBuilderInput.config.enrichPrimaryKeys) + if (databaseBuilderInput.config.enrichPrimaryKeys) { table.primaryKey = buildPrimaryKeys(catalog, schema, table, metaData); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/DatabaseBuilderConfig.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/DatabaseBuilderConfig.java similarity index 94% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/DatabaseBuilderConfig.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/DatabaseBuilderConfig.java index 7c938710dd8..4b0dddd686f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/DatabaseBuilderConfig.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/DatabaseBuilderConfig.java @@ -1,4 +1,4 @@ -// Copyright 2021 Goldman Sachs +// Copyright 2024 Goldman Sachs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,8 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// -package org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model; +package org.finos.legend.engine.plan.execution.stores.relational.exploration.model; import org.eclipse.collections.api.factory.Lists; diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/DatabaseBuilderInput.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/DatabaseBuilderInput.java similarity index 94% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/DatabaseBuilderInput.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/DatabaseBuilderInput.java index 9d268e1e644..8172931a1fd 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/DatabaseBuilderInput.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/DatabaseBuilderInput.java @@ -1,4 +1,4 @@ -// Copyright 2021 Goldman Sachs +// Copyright 2024 Goldman Sachs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,8 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// -package org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model; +package org.finos.legend.engine.plan.execution.stores.relational.exploration.model; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/DatabasePattern.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/DatabasePattern.java similarity index 97% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/DatabasePattern.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/DatabasePattern.java index 028d8f35b65..f5733ec9ce0 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/DatabasePattern.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/DatabasePattern.java @@ -1,4 +1,4 @@ -// Copyright 2021 Goldman Sachs +// Copyright 2024 Goldman Sachs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,8 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// -package org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model; +package org.finos.legend.engine.plan.execution.stores.relational.exploration.model; import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/TargetDatabase.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/TargetDatabase.java similarity index 78% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/TargetDatabase.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/TargetDatabase.java index d328f031b1a..18b3cb65c81 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/model/TargetDatabase.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/model/TargetDatabase.java @@ -1,4 +1,4 @@ -// Copyright 2021 Goldman Sachs +// Copyright 2024 Goldman Sachs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,8 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// -package org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model; +package org.finos.legend.engine.plan.execution.stores.relational.exploration.model; import com.fasterxml.jackson.annotation.JsonProperty; @@ -24,4 +25,15 @@ public class TargetDatabase @JsonProperty(value = "package") public String _package; + + public TargetDatabase() + { + + } + + public TargetDatabase(String _package, String name) + { + this.name = name; + this._package = _package; + } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/TestSchemaExploration.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/TestSchemaExploration.java similarity index 98% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/TestSchemaExploration.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/TestSchemaExploration.java index fc350366cfe..5508ccbe1d1 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection-http-api/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/connection/api/schema/TestSchemaExploration.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/exploration/TestSchemaExploration.java @@ -1,4 +1,4 @@ -// Copyright 2021 Goldman Sachs +// Copyright 2024 Goldman Sachs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,9 +11,10 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// -package org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema; +package org.finos.legend.engine.plan.execution.stores.relational.exploration; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; @@ -21,9 +22,9 @@ import org.eclipse.collections.impl.list.mutable.FastList; import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.engine.plan.execution.stores.relational.config.TemporaryTestDbConfiguration; -import org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model.DatabaseBuilderInput; -import org.finos.legend.engine.plan.execution.stores.relational.connection.api.schema.model.DatabasePattern; import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.model.DatabaseBuilderInput; +import org.finos.legend.engine.plan.execution.stores.relational.exploration.model.DatabasePattern; import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; @@ -45,7 +46,6 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.datatype.TinyInt; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.datatype.VarChar; import org.finos.legend.engine.shared.core.identity.Identity; -import org.finos.legend.engine.shared.core.identity.factory.*; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -201,8 +201,8 @@ public void testFilterTablesAndSchemasExact() throws Exception private void test(DatabaseBuilderInput input, Database expected) throws Exception { - SchemaExportation builder = SchemaExportation.newBuilder(input); - Database store = builder.build(this.connectionManager, Identity.getAnonymousIdentity()); + SchemaExportation builder = SchemaExportation.newBuilder(this.connectionManager); + Database store = builder.build(input, Identity.getAnonymousIdentity()); // Ignore the INFORMATION_SCHEMA that is added by new H2 for testing store.schemas.removeIf(s -> s.name.equals("INFORMATION_SCHEMA"));