Create namespace metadata when importing table

scalar-labs · Nov 6, 2023 · b02222a · b02222a
1 parent 85ef31f
commit b02222a
Show file tree

Hide file tree

Showing 28 changed files with 317 additions and 98 deletions.
diff --git a/...integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTableIntegrationTest.java b/...integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTableIntegrationTest.java
@@ -22,6 +22,12 @@ protected Properties getProperties(String testName) {
     return JdbcEnv.getProperties(testName);
   }
 
+  @Override
+  protected void afterAll() throws Exception {
+    super.afterAll();
+    testUtils.close();
+  }
+
   @Override
   protected Map<String, TableMetadata> createExistingDatabaseWithAllDataTypes()
       throws SQLException {

diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java
@@ -107,18 +107,20 @@ public class JdbcAdminImportTestUtils {
           "geometry",
           "geography");
 
-  private final JdbcConfig config;
   private final RdbEngineStrategy rdbEngine;
   private final int majorVersion;
+  private final BasicDataSource dataSource;
 
   public JdbcAdminImportTestUtils(Properties properties) {
-    config = new JdbcConfig(new DatabaseConfig(properties));
+    JdbcConfig config = new JdbcConfig(new DatabaseConfig(properties));
     rdbEngine = RdbEngineFactory.create(config);
+    dataSource = JdbcUtils.initDataSourceForAdmin(config, rdbEngine);
     majorVersion = getMajorVersion();
   }
 
   public Map<String, TableMetadata> createExistingDatabaseWithAllDataTypes(String namespace)
       throws SQLException {
+    execute(rdbEngine.createSchemaSqls(namespace));
     if (rdbEngine instanceof RdbEngineMysql) {
       return createExistingMysqlDatabaseWithAllDataTypes(namespace);
     } else if (rdbEngine instanceof RdbEnginePostgresql) {
@@ -138,15 +140,13 @@ public void dropTable(String namespace, String table) throws SQLException {
   }
 
   public void execute(String sql) throws SQLException {
-    try (BasicDataSource dataSource = JdbcUtils.initDataSourceForAdmin(config, rdbEngine);
-        Connection connection = dataSource.getConnection()) {
+    try (Connection connection = dataSource.getConnection()) {
       JdbcAdmin.execute(connection, sql);
     }
   }
 
   public void execute(String[] sql) throws SQLException {
-    try (BasicDataSource dataSource = JdbcUtils.initDataSourceForAdmin(config, rdbEngine);
-        Connection connection = dataSource.getConnection()) {
+    try (Connection connection = dataSource.getConnection()) {
       JdbcAdmin.execute(connection, sql);
     }
   }
@@ -539,8 +539,7 @@ private Map<String, TableMetadata> executeCreateTableSql(
   }
 
   private boolean isMariaDB() {
-    try (BasicDataSource dataSource = JdbcUtils.initDataSourceForAdmin(config, rdbEngine);
-        Connection connection = dataSource.getConnection()) {
+    try (Connection connection = dataSource.getConnection()) {
       String version = connection.getMetaData().getDatabaseProductVersion();
       return version.contains("MariaDB");
     } catch (SQLException e) {
@@ -549,11 +548,14 @@ private boolean isMariaDB() {
   }
 
   private int getMajorVersion() {
-    try (BasicDataSource dataSource = JdbcUtils.initDataSourceForAdmin(config, rdbEngine);
-        Connection connection = dataSource.getConnection()) {
+    try (Connection connection = dataSource.getConnection()) {
       return connection.getMetaData().getDatabaseMajorVersion();
     } catch (SQLException e) {
       throw new RuntimeException("Get database major version failed");
     }
   }
+
+  public void close() throws SQLException {
+    dataSource.close();
+  }
 }
diff --git a/...tegration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportIntegrationTest.java b/...tegration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportIntegrationTest.java
@@ -2,7 +2,6 @@
 
 import com.scalar.db.config.DatabaseConfig;
 import com.scalar.db.schemaloader.SchemaLoaderImportIntegrationTestBase;
-import com.scalar.db.util.AdminTestUtils;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.util.Properties;
 import org.junit.jupiter.api.Test;
@@ -23,11 +22,6 @@ protected Properties getProperties(String testName) {
     return properties;
   }
 
-  @Override
-  protected AdminTestUtils getAdminTestUtils(String testName) {
-    return new JdbcAdminTestUtils(getProperties(testName));
-  }
-
   @SuppressFBWarnings("SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE")
   @Override
   protected void createImportableTable(String namespace, String table) throws Exception {
@@ -70,6 +64,12 @@ public void importTables_ImportableTablesGiven_ShouldImportProperly() throws Exc
     super.importTables_ImportableTablesGiven_ShouldImportProperly();
   }
 
+  @Override
+  public void afterAll() throws Exception {
+    super.afterAll();
+    testUtils.close();
+  }
+
   @SuppressWarnings("unused")
   private boolean isSqlite() {
     return JdbcEnv.isSqlite();

diff --git a/core/src/main/java/com/scalar/db/api/Admin.java b/core/src/main/java/com/scalar/db/api/Admin.java
@@ -418,11 +418,13 @@ void addNewColumnToTable(String namespace, String table, String columnName, Data
    *
    * @param namespace an existing namespace
    * @param table an existing table
+   * @param options options to import
    * @throws IllegalArgumentException if the table is already managed by ScalarDB, if the target
    *     table does not exist, or if the table does not meet the requirement of ScalarDB table
    * @throws ExecutionException if the operation fails
    */
-  void importTable(String namespace, String table) throws ExecutionException;
+  void importTable(String namespace, String table, Map<String, String> options)
+      throws ExecutionException;
 
   /**
    * Returns the names of the existing namespaces created through Scalar DB.

diff --git a/core/src/main/java/com/scalar/db/common/CheckedDistributedStorageAdmin.java b/core/src/main/java/com/scalar/db/common/CheckedDistributedStorageAdmin.java
@@ -298,15 +298,16 @@ public TableMetadata getImportTableMetadata(String namespace, String table)
   }
 
   @Override
-  public void importTable(String namespace, String table) throws ExecutionException {
+  public void importTable(String namespace, String table, Map<String, String> options)
+      throws ExecutionException {
     TableMetadata tableMetadata = getTableMetadata(namespace, table);
     if (tableMetadata != null) {
       throw new IllegalArgumentException(
           "Table already exists: " + ScalarDbUtils.getFullTableName(namespace, table));
     }
 
     try {
-      admin.importTable(namespace, table);
+      admin.importTable(namespace, table, options);
     } catch (ExecutionException e) {
       throw new ExecutionException(
           "Importing the table failed: " + ScalarDbUtils.getFullTableName(namespace, table), e);

diff --git a/core/src/main/java/com/scalar/db/service/AdminService.java b/core/src/main/java/com/scalar/db/service/AdminService.java
@@ -107,8 +107,9 @@ public void addRawColumnToTable(
   }
 
   @Override
-  public void importTable(String namespace, String table) throws ExecutionException {
-    admin.importTable(namespace, table);
+  public void importTable(String namespace, String table, Map<String, String> options)
+      throws ExecutionException {
+    admin.importTable(namespace, table, options);
   }
 
   @Override

diff --git a/core/src/main/java/com/scalar/db/storage/cassandra/CassandraAdmin.java b/core/src/main/java/com/scalar/db/storage/cassandra/CassandraAdmin.java
@@ -277,7 +277,7 @@ public void addRawColumnToTable(
   }
 
   @Override
-  public void importTable(String namespace, String table) {
+  public void importTable(String namespace, String table, Map<String, String> options) {
     throw new UnsupportedOperationException(
         "Import-related functionality is not supported in Cassandra");
   }

diff --git a/core/src/main/java/com/scalar/db/storage/cosmos/CosmosAdmin.java b/core/src/main/java/com/scalar/db/storage/cosmos/CosmosAdmin.java
@@ -622,7 +622,7 @@ public void addRawColumnToTable(
   }
 
   @Override
-  public void importTable(String namespace, String table) {
+  public void importTable(String namespace, String table, Map<String, String> options) {
     throw new UnsupportedOperationException(
         "Import-related functionality is not supported in Cosmos DB");
   }

diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/DynamoAdmin.java b/core/src/main/java/com/scalar/db/storage/dynamo/DynamoAdmin.java
@@ -1328,7 +1328,7 @@ public void addRawColumnToTable(
   }
 
   @Override
-  public void importTable(String namespace, String table) {
+  public void importTable(String namespace, String table, Map<String, String> options) {
     throw new UnsupportedOperationException(
         "Import-related functionality is not supported in DynamoDB");
   }

diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcAdmin.java b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcAdmin.java
@@ -543,9 +543,12 @@ public TableMetadata getImportTableMetadata(String namespace, String table)
   }
 
   @Override
-  public void importTable(String namespace, String table) throws ExecutionException {
+  public void importTable(String namespace, String table, Map<String, String> options)
+      throws ExecutionException {
     try (Connection connection = dataSource.getConnection()) {
       TableMetadata tableMetadata = getImportTableMetadata(namespace, table);
+      createNamespacesTableIfNotExists(connection);
+      upsertIntoNamespacesTable(connection, namespace);
       addTableMetadata(connection, namespace, table, tableMetadata, true, false);
     } catch (SQLException | ExecutionException e) {
       throw new ExecutionException(
@@ -763,7 +766,7 @@ public void repairNamespace(String namespace, Map<String, String> options)
     try (Connection connection = dataSource.getConnection()) {
       createSchemaIfNotExists(connection, namespace);
       createNamespacesTableIfNotExists(connection);
-      upsertIntoNamespaceTable(connection, namespace);
+      upsertIntoNamespacesTable(connection, namespace);
     } catch (SQLException e) {
       throw new ExecutionException(String.format("Repairing the %s schema failed", namespace), e);
     }
@@ -942,7 +945,8 @@ public Set<String> getNamespaceNames() throws ExecutionException {
     }
   }
 
-  private void createNamespacesTableIfNotExists(Connection connection) throws ExecutionException {
+  @VisibleForTesting
+  void createNamespacesTableIfNotExists(Connection connection) throws ExecutionException {
     try {
       createSchemaIfNotExists(connection, metadataSchema);
       String createTableStatement =
@@ -972,8 +976,8 @@ private void insertIntoNamespacesTable(Connection connection, String namespaceNa
     }
   }
 
-  private void upsertIntoNamespaceTable(Connection connection, String namespace)
-      throws SQLException {
+  @VisibleForTesting
+  void upsertIntoNamespacesTable(Connection connection, String namespace) throws SQLException {
     try {
       insertIntoNamespacesTable(connection, namespace);
     } catch (SQLException e) {
@@ -1042,7 +1046,7 @@ private void importNamespaceNamesOfExistingTables(Connection connection)
         namespaceOfExistingTables.add(namespaceName);
       }
       for (String namespace : namespaceOfExistingTables) {
-        upsertIntoNamespaceTable(connection, namespace);
+        upsertIntoNamespacesTable(connection, namespace);
       }
     } catch (SQLException e) {
       throw new ExecutionException("Importing the namespace names of existing tables failed", e);

diff --git a/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageAdmin.java b/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageAdmin.java
@@ -209,8 +209,9 @@ public void addRawColumnToTable(
   }
 
   @Override
-  public void importTable(String namespace, String table) throws ExecutionException {
-    getAdmin(namespace, table).importTable(namespace, table);
+  public void importTable(String namespace, String table, Map<String, String> options)
+      throws ExecutionException {
+    getAdmin(namespace, table).importTable(namespace, table, options);
   }
 
   @Override

diff --git a/core/src/main/java/com/scalar/db/storage/rpc/GrpcAdmin.java b/core/src/main/java/com/scalar/db/storage/rpc/GrpcAdmin.java
@@ -340,7 +340,7 @@ public void addRawColumnToTable(
   }
 
   @Override
-  public void importTable(String namespace, String table) {
+  public void importTable(String namespace, String table, Map<String, String> options) {
     throw new UnsupportedOperationException(
         "Import-related functionality is not supported in ScalarDB Server");
   }

diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java
@@ -6,7 +6,6 @@
 import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.removeTransactionMetaColumns;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
 import com.google.inject.Inject;
 import com.scalar.db.api.DistributedStorageAdmin;
 import com.scalar.db.api.DistributedTransactionAdmin;
@@ -208,7 +207,8 @@ public Set<String> getNamespaceNames() throws ExecutionException {
   }
 
   @Override
-  public void importTable(String namespace, String table) throws ExecutionException {
+  public void importTable(String namespace, String table, Map<String, String> options)
+      throws ExecutionException {
     TableMetadata tableMetadata = getTableMetadata(namespace, table);
     if (tableMetadata != null) {
       throw new IllegalArgumentException(
@@ -231,8 +231,8 @@ public void importTable(String namespace, String table) throws ExecutionExceptio
     }
 
     // add ScalarDB metadata
-    admin.repairTable(
-        namespace, table, buildTransactionTableMetadata(tableMetadata), ImmutableMap.of());
+    admin.repairNamespace(namespace, options);
+    admin.repairTable(namespace, table, buildTransactionTableMetadata(tableMetadata), options);
   }
 
   @Override

diff --git a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdmin.java b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdmin.java
@@ -86,8 +86,9 @@ public boolean namespaceExists(String namespace) throws ExecutionException {
   }
 
   @Override
-  public void importTable(String namespace, String table) throws ExecutionException {
-    jdbcAdmin.importTable(namespace, table);
+  public void importTable(String namespace, String table, Map<String, String> options)
+      throws ExecutionException {
+    jdbcAdmin.importTable(namespace, table, options);
   }
 
   @Override

diff --git a/core/src/main/java/com/scalar/db/transaction/rpc/GrpcTransactionAdmin.java b/core/src/main/java/com/scalar/db/transaction/rpc/GrpcTransactionAdmin.java
@@ -395,7 +395,7 @@ public void addNewColumnToTable(
   }
 
   @Override
-  public void importTable(String namespace, String table) {
+  public void importTable(String namespace, String table, Map<String, String> options) {
     throw new UnsupportedOperationException(
         "Import-related functionality is not supported in ScalarDB Server");
   }

diff --git a/core/src/test/java/com/scalar/db/storage/cassandra/CassandraAdminTestBase.java b/core/src/test/java/com/scalar/db/storage/cassandra/CassandraAdminTestBase.java
@@ -831,7 +831,8 @@ public void unsupportedOperations_ShouldThrowUnsupportedException() {
     Throwable thrown2 =
         catchThrowable(
             () -> cassandraAdmin.addRawColumnToTable(namespace, table, column, DataType.INT));
-    Throwable thrown3 = catchThrowable(() -> cassandraAdmin.importTable(namespace, table));
+    Throwable thrown3 =
+        catchThrowable(() -> cassandraAdmin.importTable(namespace, table, Collections.emptyMap()));
 
     // Assert
     assertThat(thrown1).isInstanceOf(UnsupportedOperationException.class);

diff --git a/core/src/test/java/com/scalar/db/storage/cosmos/CosmosAdminTestBase.java b/core/src/test/java/com/scalar/db/storage/cosmos/CosmosAdminTestBase.java
@@ -987,7 +987,8 @@ public void unsupportedOperations_ShouldThrowUnsupportedException() {
     Throwable thrown1 = catchThrowable(() -> admin.getImportTableMetadata(namespace, table));
     Throwable thrown2 =
         catchThrowable(() -> admin.addRawColumnToTable(namespace, table, column, DataType.INT));
-    Throwable thrown3 = catchThrowable(() -> admin.importTable(namespace, table));
+    Throwable thrown3 =
+        catchThrowable(() -> admin.importTable(namespace, table, Collections.emptyMap()));
 
     // Assert
     assertThat(thrown1).isInstanceOf(UnsupportedOperationException.class);

diff --git a/core/src/test/java/com/scalar/db/storage/dynamo/DynamoAdminTestBase.java b/core/src/test/java/com/scalar/db/storage/dynamo/DynamoAdminTestBase.java
@@ -1526,7 +1526,8 @@ public void unsupportedOperations_ShouldThrowUnsupportedException() {
     Throwable thrown1 = catchThrowable(() -> admin.getImportTableMetadata(NAMESPACE, TABLE));
     Throwable thrown2 =
         catchThrowable(() -> admin.addRawColumnToTable(NAMESPACE, TABLE, "c1", DataType.INT));
-    Throwable thrown3 = catchThrowable(() -> admin.importTable(NAMESPACE, TABLE));
+    Throwable thrown3 =
+        catchThrowable(() -> admin.importTable(NAMESPACE, TABLE, Collections.emptyMap()));
 
     // Assert
     assertThat(thrown1).isInstanceOf(UnsupportedOperationException.class);