From 51274f3bbb73b3175e67183273cd959415f95575 Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Thu, 30 Nov 2023 15:28:19 +0900 Subject: [PATCH] Add options to Admin.importTable() (#1337) --- .../main/java/com/scalar/db/api/Admin.java | 4 +- .../CheckedDistributedStorageAdmin.java | 5 +- .../com/scalar/db/service/AdminService.java | 5 +- .../db/storage/cassandra/CassandraAdmin.java | 2 +- .../scalar/db/storage/cosmos/CosmosAdmin.java | 2 +- .../scalar/db/storage/dynamo/DynamoAdmin.java | 2 +- .../com/scalar/db/storage/jdbc/JdbcAdmin.java | 6 +- .../multistorage/MultiStorageAdmin.java | 5 +- .../com/scalar/db/storage/rpc/GrpcAdmin.java | 2 +- .../consensuscommit/ConsensusCommitAdmin.java | 7 +- .../jdbc/JdbcTransactionAdmin.java | 5 +- .../transaction/rpc/GrpcTransactionAdmin.java | 2 +- .../storage/cassandra/CassandraAdminTest.java | 3 +- .../storage/cosmos/CosmosAdminTestBase.java | 3 +- .../storage/dynamo/DynamoAdminTestBase.java | 3 +- .../db/storage/jdbc/JdbcAdminTestBase.java | 5 +- .../scalar/db/storage/rpc/GrpcAdminTest.java | 19 ++-- .../ConsensusCommitAdminTestBase.java | 12 ++- .../jdbc/JdbcTransactionAdminTest.java | 4 +- .../rpc/GrpcTransactionAdminTest.java | 12 +++ ...geAdminImportTableIntegrationTestBase.java | 10 ++- ...onAdminImportTableIntegrationTestBase.java | 10 ++- .../db/schemaloader/ImportSchemaParser.java | 12 ++- .../db/schemaloader/ImportTableSchema.java | 36 +++++++- .../scalar/db/schemaloader/SchemaLoader.java | 45 ++++++---- .../db/schemaloader/SchemaOperator.java | 7 +- .../scalar/db/schemaloader/TableSchema.java | 10 +-- .../command/SchemaLoaderCommand.java | 88 ++++++++++++------- .../schemaloader/ImportSchemaParserTest.java | 15 +++- .../schemaloader/ImportTableSchemaTest.java | 44 +++++++++- .../db/schemaloader/SchemaLoaderTest.java | 19 ++-- .../db/schemaloader/SchemaOperatorTest.java | 8 +- .../command/SchemaLoaderCommandTest.java | 28 +++++- 33 files changed, 306 insertions(+), 134 deletions(-) diff --git a/core/src/main/java/com/scalar/db/api/Admin.java b/core/src/main/java/com/scalar/db/api/Admin.java index e75774606a..93a082ce4e 100644 --- a/core/src/main/java/com/scalar/db/api/Admin.java +++ b/core/src/main/java/com/scalar/db/api/Admin.java @@ -408,9 +408,11 @@ void addNewColumnToTable(String namespace, String table, String columnName, Data * * @param namespace an existing namespace * @param table an existing table + * @param options options to import * @throws IllegalArgumentException if the table is already managed by ScalarDB, if the target * table does not exist, or if the table does not meet the requirement of ScalarDB table * @throws ExecutionException if the operation fails */ - void importTable(String namespace, String table) throws ExecutionException; + void importTable(String namespace, String table, Map options) + throws ExecutionException; } diff --git a/core/src/main/java/com/scalar/db/common/CheckedDistributedStorageAdmin.java b/core/src/main/java/com/scalar/db/common/CheckedDistributedStorageAdmin.java index 514f4d202d..e1425560d4 100644 --- a/core/src/main/java/com/scalar/db/common/CheckedDistributedStorageAdmin.java +++ b/core/src/main/java/com/scalar/db/common/CheckedDistributedStorageAdmin.java @@ -290,7 +290,8 @@ public TableMetadata getImportTableMetadata(String namespace, String table) } @Override - public void importTable(String namespace, String table) throws ExecutionException { + public void importTable(String namespace, String table, Map options) + throws ExecutionException { TableMetadata tableMetadata = getTableMetadata(namespace, table); if (tableMetadata != null) { throw new IllegalArgumentException( @@ -298,7 +299,7 @@ public void importTable(String namespace, String table) throws ExecutionExceptio } try { - admin.importTable(namespace, table); + admin.importTable(namespace, table, options); } catch (ExecutionException e) { throw new ExecutionException( "Importing the table failed: " + ScalarDbUtils.getFullTableName(namespace, table), e); diff --git a/core/src/main/java/com/scalar/db/service/AdminService.java b/core/src/main/java/com/scalar/db/service/AdminService.java index 14d339785e..9c30e7ff8c 100644 --- a/core/src/main/java/com/scalar/db/service/AdminService.java +++ b/core/src/main/java/com/scalar/db/service/AdminService.java @@ -107,8 +107,9 @@ public void addRawColumnToTable( } @Override - public void importTable(String namespace, String table) throws ExecutionException { - admin.importTable(namespace, table); + public void importTable(String namespace, String table, Map options) + throws ExecutionException { + admin.importTable(namespace, table, options); } @Override diff --git a/core/src/main/java/com/scalar/db/storage/cassandra/CassandraAdmin.java b/core/src/main/java/com/scalar/db/storage/cassandra/CassandraAdmin.java index 7461bbc4de..4ba92a358f 100644 --- a/core/src/main/java/com/scalar/db/storage/cassandra/CassandraAdmin.java +++ b/core/src/main/java/com/scalar/db/storage/cassandra/CassandraAdmin.java @@ -201,7 +201,7 @@ public void addRawColumnToTable( } @Override - public void importTable(String namespace, String table) { + public void importTable(String namespace, String table, Map options) { throw new UnsupportedOperationException( "Import-related functionality is not supported in Cassandra"); } diff --git a/core/src/main/java/com/scalar/db/storage/cosmos/CosmosAdmin.java b/core/src/main/java/com/scalar/db/storage/cosmos/CosmosAdmin.java index f525af7f86..f2ebb3b089 100644 --- a/core/src/main/java/com/scalar/db/storage/cosmos/CosmosAdmin.java +++ b/core/src/main/java/com/scalar/db/storage/cosmos/CosmosAdmin.java @@ -566,7 +566,7 @@ public void addRawColumnToTable( } @Override - public void importTable(String namespace, String table) { + public void importTable(String namespace, String table, Map options) { throw new UnsupportedOperationException( "Import-related functionality is not supported in Cosmos DB"); } diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/DynamoAdmin.java b/core/src/main/java/com/scalar/db/storage/dynamo/DynamoAdmin.java index 2407b7cec9..97ad1c2238 100644 --- a/core/src/main/java/com/scalar/db/storage/dynamo/DynamoAdmin.java +++ b/core/src/main/java/com/scalar/db/storage/dynamo/DynamoAdmin.java @@ -1214,7 +1214,7 @@ public void addRawColumnToTable( } @Override - public void importTable(String namespace, String table) { + public void importTable(String namespace, String table, Map options) { throw new UnsupportedOperationException( "Import-related functionality is not supported in DynamoDB"); } diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcAdmin.java b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcAdmin.java index 4698f828bc..c5d37fb12a 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcAdmin.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcAdmin.java @@ -5,7 +5,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.inject.Inject; @@ -490,11 +489,12 @@ public TableMetadata getImportTableMetadata(String namespace, String table) } @Override - public void importTable(String namespace, String table) throws ExecutionException { + public void importTable(String namespace, String table, Map options) + throws ExecutionException { TableMetadata tableMetadata = getImportTableMetadata(namespace, table); // add ScalarDB metadata - repairTable(namespace, table, tableMetadata, ImmutableMap.of()); + repairTable(namespace, table, tableMetadata, options); } private String getSelectColumnsStatement() { diff --git a/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageAdmin.java b/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageAdmin.java index 7afb592da9..36afe8c7e0 100644 --- a/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageAdmin.java +++ b/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageAdmin.java @@ -196,8 +196,9 @@ public void addRawColumnToTable( } @Override - public void importTable(String namespace, String table) throws ExecutionException { - getAdmin(namespace, table).importTable(namespace, table); + public void importTable(String namespace, String table, Map options) + throws ExecutionException { + getAdmin(namespace, table).importTable(namespace, table, options); } private DistributedStorageAdmin getAdmin(String namespace) { diff --git a/core/src/main/java/com/scalar/db/storage/rpc/GrpcAdmin.java b/core/src/main/java/com/scalar/db/storage/rpc/GrpcAdmin.java index 2b559fe140..80506871b6 100644 --- a/core/src/main/java/com/scalar/db/storage/rpc/GrpcAdmin.java +++ b/core/src/main/java/com/scalar/db/storage/rpc/GrpcAdmin.java @@ -340,7 +340,7 @@ public void addRawColumnToTable( } @Override - public void importTable(String namespace, String table) { + public void importTable(String namespace, String table, Map options) { throw new UnsupportedOperationException( "Import-related functionality is not supported in ScalarDB Server"); } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java index 285d65cf04..6dadf445b9 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java @@ -6,7 +6,6 @@ import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.removeTransactionMetaColumns; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.scalar.db.api.DistributedStorageAdmin; import com.scalar.db.api.DistributedTransactionAdmin; @@ -193,7 +192,8 @@ public void addNewColumnToTable( } @Override - public void importTable(String namespace, String table) throws ExecutionException { + public void importTable(String namespace, String table, Map options) + throws ExecutionException { TableMetadata tableMetadata = admin.getTableMetadata(namespace, table); if (tableMetadata != null) { throw new IllegalArgumentException( @@ -216,8 +216,7 @@ public void importTable(String namespace, String table) throws ExecutionExceptio } // add ScalarDB metadata - admin.repairTable( - namespace, table, buildTransactionTableMetadata(tableMetadata), ImmutableMap.of()); + admin.repairTable(namespace, table, buildTransactionTableMetadata(tableMetadata), options); } @Override diff --git a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdmin.java b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdmin.java index 6fe69e8a6e..6537c09166 100644 --- a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdmin.java +++ b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdmin.java @@ -91,8 +91,9 @@ public boolean namespaceExists(String namespace) throws ExecutionException { } @Override - public void importTable(String namespace, String table) throws ExecutionException { - jdbcAdmin.importTable(namespace, table); + public void importTable(String namespace, String table, Map options) + throws ExecutionException { + jdbcAdmin.importTable(namespace, table, options); } @Override diff --git a/core/src/main/java/com/scalar/db/transaction/rpc/GrpcTransactionAdmin.java b/core/src/main/java/com/scalar/db/transaction/rpc/GrpcTransactionAdmin.java index 5bc02c659c..8454d13502 100644 --- a/core/src/main/java/com/scalar/db/transaction/rpc/GrpcTransactionAdmin.java +++ b/core/src/main/java/com/scalar/db/transaction/rpc/GrpcTransactionAdmin.java @@ -395,7 +395,7 @@ public void addNewColumnToTable( } @Override - public void importTable(String namespace, String table) { + public void importTable(String namespace, String table, Map options) { throw new UnsupportedOperationException( "Import-related functionality is not supported in ScalarDB Server"); } diff --git a/core/src/test/java/com/scalar/db/storage/cassandra/CassandraAdminTest.java b/core/src/test/java/com/scalar/db/storage/cassandra/CassandraAdminTest.java index 73e1689b0d..d709b4fab2 100644 --- a/core/src/test/java/com/scalar/db/storage/cassandra/CassandraAdminTest.java +++ b/core/src/test/java/com/scalar/db/storage/cassandra/CassandraAdminTest.java @@ -606,7 +606,8 @@ public void unsupportedOperations_ShouldThrowUnsupportedException() { Throwable thrown2 = catchThrowable( () -> cassandraAdmin.addRawColumnToTable(namespace, table, column, DataType.INT)); - Throwable thrown3 = catchThrowable(() -> cassandraAdmin.importTable(namespace, table)); + Throwable thrown3 = + catchThrowable(() -> cassandraAdmin.importTable(namespace, table, Collections.emptyMap())); // Assert assertThat(thrown1).isInstanceOf(UnsupportedOperationException.class); diff --git a/core/src/test/java/com/scalar/db/storage/cosmos/CosmosAdminTestBase.java b/core/src/test/java/com/scalar/db/storage/cosmos/CosmosAdminTestBase.java index f0f56a3dff..677a579289 100644 --- a/core/src/test/java/com/scalar/db/storage/cosmos/CosmosAdminTestBase.java +++ b/core/src/test/java/com/scalar/db/storage/cosmos/CosmosAdminTestBase.java @@ -855,7 +855,8 @@ public void unsupportedOperations_ShouldThrowUnsupportedException() { Throwable thrown1 = catchThrowable(() -> admin.getImportTableMetadata(namespace, table)); Throwable thrown2 = catchThrowable(() -> admin.addRawColumnToTable(namespace, table, column, DataType.INT)); - Throwable thrown3 = catchThrowable(() -> admin.importTable(namespace, table)); + Throwable thrown3 = + catchThrowable(() -> admin.importTable(namespace, table, Collections.emptyMap())); // Assert assertThat(thrown1).isInstanceOf(UnsupportedOperationException.class); diff --git a/core/src/test/java/com/scalar/db/storage/dynamo/DynamoAdminTestBase.java b/core/src/test/java/com/scalar/db/storage/dynamo/DynamoAdminTestBase.java index d8dca58715..bf57858e48 100644 --- a/core/src/test/java/com/scalar/db/storage/dynamo/DynamoAdminTestBase.java +++ b/core/src/test/java/com/scalar/db/storage/dynamo/DynamoAdminTestBase.java @@ -1183,7 +1183,8 @@ public void unsupportedOperations_ShouldThrowUnsupportedException() { Throwable thrown1 = catchThrowable(() -> admin.getImportTableMetadata(NAMESPACE, TABLE)); Throwable thrown2 = catchThrowable(() -> admin.addRawColumnToTable(NAMESPACE, TABLE, "c1", DataType.INT)); - Throwable thrown3 = catchThrowable(() -> admin.importTable(NAMESPACE, TABLE)); + Throwable thrown3 = + catchThrowable(() -> admin.importTable(NAMESPACE, TABLE, Collections.emptyMap())); // Assert assertThat(thrown1).isInstanceOf(UnsupportedOperationException.class); diff --git a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTestBase.java b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTestBase.java index 0ff6834fd6..413272bdf6 100644 --- a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTestBase.java +++ b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTestBase.java @@ -2519,7 +2519,7 @@ private void importTable_ForX_ShouldWorkProperly( JdbcAdmin admin = createJdbcAdminFor(rdbEngine); // Act - admin.importTable(NAMESPACE, TABLE); + admin.importTable(NAMESPACE, TABLE, Collections.emptyMap()); // Assert for (int i = 0; i < expectedSqlStatements.size(); i++) { @@ -2536,7 +2536,8 @@ public void importTable_ForSQLite_ShouldThrowUnsupportedOperationException() { JdbcAdmin admin = createJdbcAdminFor(RdbEngine.SQLITE); // Act - Throwable thrown = catchThrowable(() -> admin.importTable(NAMESPACE, TABLE)); + Throwable thrown = + catchThrowable(() -> admin.importTable(NAMESPACE, TABLE, Collections.emptyMap())); // Assert assertThat(thrown).isInstanceOf(UnsupportedOperationException.class); diff --git a/core/src/test/java/com/scalar/db/storage/rpc/GrpcAdminTest.java b/core/src/test/java/com/scalar/db/storage/rpc/GrpcAdminTest.java index 89d9dc0b27..9de3c74a2f 100644 --- a/core/src/test/java/com/scalar/db/storage/rpc/GrpcAdminTest.java +++ b/core/src/test/java/com/scalar/db/storage/rpc/GrpcAdminTest.java @@ -1,7 +1,7 @@ package com.scalar.db.storage.rpc; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.catchThrowable; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; @@ -340,15 +340,12 @@ public void unsupportedOperations_ShouldThrowUnsupportedException() { String table = "tbl"; String column = "col"; - // Act - Throwable thrown1 = catchThrowable(() -> admin.getImportTableMetadata(namespace, table)); - Throwable thrown2 = - catchThrowable(() -> admin.addRawColumnToTable(namespace, table, column, DataType.INT)); - Throwable thrown3 = catchThrowable(() -> admin.importTable(namespace, table)); - - // Assert - assertThat(thrown1).isInstanceOf(UnsupportedOperationException.class); - assertThat(thrown2).isInstanceOf(UnsupportedOperationException.class); - assertThat(thrown3).isInstanceOf(UnsupportedOperationException.class); + // Act Assert + assertThatThrownBy(() -> admin.getImportTableMetadata(namespace, table)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> admin.addRawColumnToTable(namespace, table, column, DataType.INT)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> admin.importTable(namespace, table, Collections.emptyMap())) + .isInstanceOf(UnsupportedOperationException.class); } } diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminTestBase.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminTestBase.java index 019e8799f4..b3354b359b 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminTestBase.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminTestBase.java @@ -1,5 +1,6 @@ package com.scalar.db.transaction.consensuscommit; +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.buildTransactionTableMetadata; import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.getBeforeImageColumnName; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -594,6 +595,7 @@ public void addNewColumnToTable_ShouldCallJdbcAdminProperly() throws ExecutionEx @Test public void importTable_ShouldCallStorageAdminProperly() throws ExecutionException { // Arrange + Map options = ImmutableMap.of("foo", "bar"); String primaryKeyColumn = "pk"; String column = "col"; TableMetadata metadata = @@ -609,7 +611,7 @@ public void importTable_ShouldCallStorageAdminProperly() throws ExecutionExcepti .addRawColumnToTable(anyString(), anyString(), anyString(), any(DataType.class)); // Act - admin.importTable(NAMESPACE, TABLE); + admin.importTable(NAMESPACE, TABLE, options); // Assert verify(distributedStorageAdmin).getTableMetadata(NAMESPACE, TABLE); @@ -624,6 +626,8 @@ public void importTable_ShouldCallStorageAdminProperly() throws ExecutionExcepti NAMESPACE, TABLE, getBeforeImageColumnName(column, metadata), DataType.INT); verify(distributedStorageAdmin, never()) .addRawColumnToTable(NAMESPACE, TABLE, primaryKeyColumn, DataType.INT); + verify(distributedStorageAdmin) + .repairTable(NAMESPACE, TABLE, buildTransactionTableMetadata(metadata), options); } @Test @@ -641,7 +645,8 @@ public void importTable_WithStorageTableAlreadyExists_ShouldThrowIllegalArgument when(distributedStorageAdmin.getTableMetadata(NAMESPACE, TABLE)).thenReturn(metadata); // Act - Throwable thrown = catchThrowable(() -> admin.importTable(NAMESPACE, TABLE)); + Throwable thrown = + catchThrowable(() -> admin.importTable(NAMESPACE, TABLE, Collections.emptyMap())); // Assert assertThat(thrown).isInstanceOf(IllegalArgumentException.class); @@ -673,7 +678,8 @@ public void importTable_WithTransactionTableAlreadyExists_ShouldThrowIllegalArgu when(distributedStorageAdmin.getTableMetadata(NAMESPACE, TABLE)).thenReturn(metadata); // Act - Throwable thrown = catchThrowable(() -> admin.importTable(NAMESPACE, TABLE)); + Throwable thrown = + catchThrowable(() -> admin.importTable(NAMESPACE, TABLE, Collections.emptyMap())); // Assert assertThat(thrown).isInstanceOf(IllegalArgumentException.class); diff --git a/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminTest.java b/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminTest.java index cd6088b5a3..6d1cfe9c54 100644 --- a/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminTest.java +++ b/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminTest.java @@ -235,9 +235,9 @@ public void importTable_ShouldCallJdbcAdminProperly() throws ExecutionException String table = "tbl"; // Act - admin.importTable(namespace, table); + admin.importTable(namespace, table, Collections.emptyMap()); // Assert - verify(jdbcAdmin).importTable(namespace, table); + verify(jdbcAdmin).importTable(namespace, table, Collections.emptyMap()); } } diff --git a/core/src/test/java/com/scalar/db/transaction/rpc/GrpcTransactionAdminTest.java b/core/src/test/java/com/scalar/db/transaction/rpc/GrpcTransactionAdminTest.java index b5bd84caa9..dcc45d5345 100644 --- a/core/src/test/java/com/scalar/db/transaction/rpc/GrpcTransactionAdminTest.java +++ b/core/src/test/java/com/scalar/db/transaction/rpc/GrpcTransactionAdminTest.java @@ -1,6 +1,7 @@ package com.scalar.db.transaction.rpc; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; @@ -411,4 +412,15 @@ public void addNewColumnToTable_CalledWithProperArguments_StubShouldBeCalledProp .setColumnType(com.scalar.db.rpc.DataType.DATA_TYPE_TEXT) .build()); } + + @Test + public void unsupportedOperations_ShouldThrowUnsupportedException() { + // Arrange + String namespace = "sample_ns"; + String table = "tbl"; + + // Act Assert + assertThatThrownBy(() -> admin.importTable(namespace, table, Collections.emptyMap())) + .isInstanceOf(UnsupportedOperationException.class); + } } diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminImportTableIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminImportTableIntegrationTestBase.java index 8ae757b631..d22b3d6407 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminImportTableIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminImportTableIntegrationTestBase.java @@ -102,14 +102,15 @@ public void importTable_ForUnsupportedDatabase_ShouldThrowUnsupportedOperationEx admin.createNamespace(getNamespace(), getCreationOptions()); // Act Assert - assertThatThrownBy(() -> admin.importTable(getNamespace(), "unsupported_db")) + assertThatThrownBy( + () -> admin.importTable(getNamespace(), "unsupported_db", Collections.emptyMap())) .isInstanceOf(UnsupportedOperationException.class); } private void importTable_ForImportableTable_ShouldImportProperly( String table, TableMetadata metadata) throws ExecutionException { // Act - admin.importTable(getNamespace(), table); + admin.importTable(getNamespace(), table, Collections.emptyMap()); // Assert assertThat(admin.tableExists(getNamespace(), table)).isTrue(); @@ -119,14 +120,15 @@ private void importTable_ForImportableTable_ShouldImportProperly( private void importTable_ForNonImportableTable_ShouldThrowIllegalArgumentException(String table) { // Act Assert assertThatThrownBy( - () -> admin.importTable(getNamespace(), table), + () -> admin.importTable(getNamespace(), table, Collections.emptyMap()), "non-importable data type test failed: " + table) .isInstanceOf(IllegalArgumentException.class); } private void importTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() { // Act Assert - assertThatThrownBy(() -> admin.importTable(getNamespace(), "non-existing-table")) + assertThatThrownBy( + () -> admin.importTable(getNamespace(), "non-existing-table", Collections.emptyMap())) .isInstanceOf(IllegalArgumentException.class); } } diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminImportTableIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminImportTableIntegrationTestBase.java index c6b059c527..64ec48d768 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminImportTableIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminImportTableIntegrationTestBase.java @@ -102,14 +102,15 @@ public void importTable_ForUnsupportedDatabase_ShouldThrowUnsupportedOperationEx admin.createNamespace(getNamespace(), getCreationOptions()); // Act Assert - assertThatThrownBy(() -> admin.importTable(getNamespace(), "unsupported_db")) + assertThatThrownBy( + () -> admin.importTable(getNamespace(), "unsupported_db", Collections.emptyMap())) .isInstanceOf(UnsupportedOperationException.class); } private void importTable_ForImportableTable_ShouldImportProperly( String table, TableMetadata metadata) throws ExecutionException { // Act - admin.importTable(getNamespace(), table); + admin.importTable(getNamespace(), table, Collections.emptyMap()); // Assert assertThat(admin.tableExists(getNamespace(), table)).isTrue(); @@ -118,13 +119,14 @@ private void importTable_ForImportableTable_ShouldImportProperly( private void importTable_ForNonImportableTable_ShouldThrowIllegalArgumentException(String table) { // Act Assert - assertThatThrownBy(() -> admin.importTable(getNamespace(), table)) + assertThatThrownBy(() -> admin.importTable(getNamespace(), table, Collections.emptyMap())) .isInstanceOf(IllegalArgumentException.class); } private void importTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() { // Act Assert - assertThatThrownBy(() -> admin.importTable(getNamespace(), "non-existing-table")) + assertThatThrownBy( + () -> admin.importTable(getNamespace(), "non-existing-table", Collections.emptyMap())) .isInstanceOf(IllegalArgumentException.class); } } diff --git a/schema-loader/src/main/java/com/scalar/db/schemaloader/ImportSchemaParser.java b/schema-loader/src/main/java/com/scalar/db/schemaloader/ImportSchemaParser.java index fe784d49e4..eb854f2a2c 100644 --- a/schema-loader/src/main/java/com/scalar/db/schemaloader/ImportSchemaParser.java +++ b/schema-loader/src/main/java/com/scalar/db/schemaloader/ImportSchemaParser.java @@ -1,5 +1,6 @@ package com.scalar.db.schemaloader; +import com.google.common.collect.ImmutableMap; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; @@ -16,21 +17,26 @@ @ThreadSafe public class ImportSchemaParser { private final JsonObject schemaJson; + private final Map options; - public ImportSchemaParser(Path jsonFilePath) throws SchemaLoaderException { + public ImportSchemaParser(Path jsonFilePath, Map options) + throws SchemaLoaderException { try (Reader reader = Files.newBufferedReader(jsonFilePath)) { schemaJson = JsonParser.parseReader(reader).getAsJsonObject(); } catch (IOException | JsonParseException e) { throw new SchemaLoaderException("Parsing the schema JSON failed", e); } + this.options = ImmutableMap.copyOf(options); } - public ImportSchemaParser(String serializedSchemaJson) throws SchemaLoaderException { + public ImportSchemaParser(String serializedSchemaJson, Map options) + throws SchemaLoaderException { try { schemaJson = JsonParser.parseString(serializedSchemaJson).getAsJsonObject(); } catch (JsonParseException e) { throw new SchemaLoaderException("Parsing the schema JSON failed", e); } + this.options = ImmutableMap.copyOf(options); } // For the SpotBugs warning CT_CONSTRUCTOR_THROW @@ -41,7 +47,7 @@ public List parse() throws SchemaLoaderException { List tableSchemaList = new ArrayList<>(); for (Map.Entry entry : schemaJson.entrySet()) { tableSchemaList.add( - new ImportTableSchema(entry.getKey(), entry.getValue().getAsJsonObject())); + new ImportTableSchema(entry.getKey(), entry.getValue().getAsJsonObject(), options)); } return tableSchemaList; } diff --git a/schema-loader/src/main/java/com/scalar/db/schemaloader/ImportTableSchema.java b/schema-loader/src/main/java/com/scalar/db/schemaloader/ImportTableSchema.java index 09570917a5..ac489af6a2 100644 --- a/schema-loader/src/main/java/com/scalar/db/schemaloader/ImportTableSchema.java +++ b/schema-loader/src/main/java/com/scalar/db/schemaloader/ImportTableSchema.java @@ -1,16 +1,21 @@ package com.scalar.db.schemaloader; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.gson.JsonObject; +import java.util.Map; +import java.util.Set; import javax.annotation.concurrent.Immutable; @Immutable public class ImportTableSchema { - private static final String TRANSACTION = "transaction"; private final String namespace; private final String tableName; private final boolean isTransactionTable; + private final ImmutableMap options; - public ImportTableSchema(String tableFullName, JsonObject tableDefinition) + public ImportTableSchema( + String tableFullName, JsonObject tableDefinition, Map options) throws SchemaLoaderException { String[] fullName = tableFullName.split("\\.", -1); if (fullName.length != 2) { @@ -20,11 +25,30 @@ public ImportTableSchema(String tableFullName, JsonObject tableDefinition) } namespace = fullName[0]; tableName = fullName[1]; - if (tableDefinition.keySet().contains(TRANSACTION)) { - isTransactionTable = tableDefinition.get(TRANSACTION).getAsBoolean(); + if (tableDefinition.keySet().contains(TableSchema.TRANSACTION)) { + isTransactionTable = tableDefinition.get(TableSchema.TRANSACTION).getAsBoolean(); } else { isTransactionTable = true; } + this.options = buildOptions(tableDefinition, options); + } + + private ImmutableMap buildOptions( + JsonObject tableDefinition, Map globalOptions) { + ImmutableMap.Builder optionsBuilder = ImmutableMap.builder(); + optionsBuilder.putAll(globalOptions); + Set keysToIgnore = + ImmutableSet.of( + TableSchema.PARTITION_KEY, + TableSchema.CLUSTERING_KEY, + TableSchema.TRANSACTION, + TableSchema.COLUMNS, + TableSchema.SECONDARY_INDEX); + tableDefinition.entrySet().stream() + .filter(entry -> !keysToIgnore.contains(entry.getKey())) + .forEach(entry -> optionsBuilder.put(entry.getKey(), entry.getValue().getAsString())); + // If an option is defined globally and in the JSON file, the JSON file value is used + return optionsBuilder.buildKeepingLast(); } // For the SpotBugs warning CT_CONSTRUCTOR_THROW @@ -42,4 +66,8 @@ public String getTable() { public boolean isTransactionTable() { return isTransactionTable; } + + public Map getOptions() { + return options; + } } diff --git a/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaLoader.java b/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaLoader.java index 18f949ee73..5dd9956d2c 100644 --- a/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaLoader.java +++ b/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaLoader.java @@ -513,13 +513,15 @@ private static void alterTables( * * @param configProperties ScalarDB config properties * @param serializedSchemaJson serialized json string schema. + * @param options specific options for importing. * @throws SchemaLoaderException thrown when importing tables fails. */ - public static void importTables(Properties configProperties, String serializedSchemaJson) + public static void importTables( + Properties configProperties, String serializedSchemaJson, Map options) throws SchemaLoaderException { Either config = new Right<>(configProperties); Either schema = new Right<>(serializedSchemaJson); - importTables(config, schema); + importTables(config, schema, options); } /** @@ -527,13 +529,15 @@ public static void importTables(Properties configProperties, String serializedSc * * @param configProperties ScalarDB properties. * @param schemaPath path to the schema file. + * @param options specific options for importing. * @throws SchemaLoaderException thrown when importing tables fails. */ - public static void importTables(Properties configProperties, Path schemaPath) + public static void importTables( + Properties configProperties, Path schemaPath, Map options) throws SchemaLoaderException { Either config = new Right<>(configProperties); Either schema = new Left<>(schemaPath); - importTables(config, schema); + importTables(config, schema, options); } /** @@ -541,13 +545,15 @@ public static void importTables(Properties configProperties, Path schemaPath) * * @param configPath path to the ScalarDB config. * @param serializedSchemaJson serialized json string schema. + * @param options specific options for importing. * @throws SchemaLoaderException thrown when importing tables fails. */ - public static void importTables(Path configPath, String serializedSchemaJson) + public static void importTables( + Path configPath, String serializedSchemaJson, Map options) throws SchemaLoaderException { Either config = new Left<>(configPath); Either schema = new Right<>(serializedSchemaJson); - importTables(config, schema); + importTables(config, schema, options); } /** @@ -555,22 +561,25 @@ public static void importTables(Path configPath, String serializedSchemaJson) * * @param configPath path to the ScalarDB config. * @param schemaPath path to the schema file. + * @param options specific options for importing. * @throws SchemaLoaderException thrown when importing tables fails. */ - public static void importTables(Path configPath, Path schemaPath) throws SchemaLoaderException { + public static void importTables(Path configPath, Path schemaPath, Map options) + throws SchemaLoaderException { Either config = new Left<>(configPath); Either schema = new Left<>(schemaPath); - importTables(config, schema); + importTables(config, schema, options); } - private static void importTables(Either config, Either schema) + private static void importTables( + Either config, Either schema, Map options) throws SchemaLoaderException { // Parse the schema - List tableSchemaList = getImportTableSchemaList(schema); + List tableSchemaList = getImportTableSchemaList(schema, options); // Import tables try (SchemaOperator operator = getSchemaOperator(config)) { - operator.importTables(tableSchemaList); + operator.importTables(tableSchemaList, options); } } @@ -612,25 +621,25 @@ static SchemaParser getSchemaParser(Either schema, Map getImportTableSchemaList(Either schema) - throws SchemaLoaderException { + private static List getImportTableSchemaList( + Either schema, Map options) throws SchemaLoaderException { if ((schema.isLeft() && schema.getLeft() != null) || (schema.isRight() && schema.getRight() != null)) { - ImportSchemaParser schemaParser = getImportSchemaParser(schema); + ImportSchemaParser schemaParser = getImportSchemaParser(schema, options); return schemaParser.parse(); } return Collections.emptyList(); } @VisibleForTesting - static ImportSchemaParser getImportSchemaParser(Either schema) - throws SchemaLoaderException { + static ImportSchemaParser getImportSchemaParser( + Either schema, Map options) throws SchemaLoaderException { assert (schema.isLeft() && schema.getLeft() != null) || (schema.isRight() && schema.getRight() != null); if (schema.isLeft()) { - return new ImportSchemaParser(schema.getLeft()); + return new ImportSchemaParser(schema.getLeft(), options); } else { - return new ImportSchemaParser(schema.getRight()); + return new ImportSchemaParser(schema.getRight(), options); } } } diff --git a/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaOperator.java b/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaOperator.java index 8fc030f8c6..1a12eeeaf8 100644 --- a/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaOperator.java +++ b/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaOperator.java @@ -367,15 +367,16 @@ private void addNewColumnsToTable( } } - public void importTables(List tableSchemaList) throws SchemaLoaderException { + public void importTables(List tableSchemaList, Map options) + throws SchemaLoaderException { for (ImportTableSchema tableSchema : tableSchemaList) { String namespace = tableSchema.getNamespace(); String table = tableSchema.getTable(); try { if (tableSchema.isTransactionTable()) { - transactionAdmin.get().importTable(namespace, table); + transactionAdmin.get().importTable(namespace, table, options); } else { - storageAdmin.get().importTable(namespace, table); + storageAdmin.get().importTable(namespace, table, options); } logger.info("Importing the table {} in the namespace {} succeeded", table, namespace); } catch (ExecutionException e) { diff --git a/schema-loader/src/main/java/com/scalar/db/schemaloader/TableSchema.java b/schema-loader/src/main/java/com/scalar/db/schemaloader/TableSchema.java index 4d9d0aad70..cacfd5b32c 100644 --- a/schema-loader/src/main/java/com/scalar/db/schemaloader/TableSchema.java +++ b/schema-loader/src/main/java/com/scalar/db/schemaloader/TableSchema.java @@ -20,11 +20,11 @@ @Immutable public class TableSchema { - private static final String COLUMNS = "columns"; - private static final String TRANSACTION = "transaction"; - private static final String PARTITION_KEY = "partition-key"; - private static final String CLUSTERING_KEY = "clustering-key"; - private static final String SECONDARY_INDEX = "secondary-index"; + static final String COLUMNS = "columns"; + static final String TRANSACTION = "transaction"; + static final String PARTITION_KEY = "partition-key"; + static final String CLUSTERING_KEY = "clustering-key"; + static final String SECONDARY_INDEX = "secondary-index"; private static final ImmutableMap DATA_MAP_TYPE = ImmutableMap.builder() .put("BOOLEAN", DataType.BOOLEAN) diff --git a/schema-loader/src/main/java/com/scalar/db/schemaloader/command/SchemaLoaderCommand.java b/schema-loader/src/main/java/com/scalar/db/schemaloader/command/SchemaLoaderCommand.java index 9adbf0c8ba..0c8588e97e 100644 --- a/schema-loader/src/main/java/com/scalar/db/schemaloader/command/SchemaLoaderCommand.java +++ b/schema-loader/src/main/java/com/scalar/db/schemaloader/command/SchemaLoaderCommand.java @@ -1,5 +1,6 @@ package com.scalar.db.schemaloader.command; +import com.google.common.collect.ImmutableMap; import com.scalar.db.schemaloader.SchemaLoader; import com.scalar.db.schemaloader.SchemaLoaderException; import com.scalar.db.storage.cassandra.CassandraAdmin; @@ -7,7 +8,6 @@ import com.scalar.db.storage.cassandra.CassandraAdmin.ReplicationStrategy; import com.scalar.db.storage.dynamo.DynamoAdmin; import java.nio.file.Path; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; import org.slf4j.Logger; @@ -36,7 +36,7 @@ public class SchemaLoaderCommand implements Callable { @Option( names = "--replication-factor", description = "The replication factor (supported in Cassandra)") - private String replicaFactor; + private String replicationFactor; @Option(names = "--ru", description = "Base resource unit (supported in DynamoDB, Cosmos DB)") private String ru; @@ -118,26 +118,7 @@ public Integer call() throws Exception { } private void createTables() throws SchemaLoaderException { - Map options = new HashMap<>(); - if (replicationStrategy != null) { - options.put(CassandraAdmin.REPLICATION_STRATEGY, replicationStrategy.toString()); - } - if (compactionStrategy != null) { - options.put(CassandraAdmin.COMPACTION_STRATEGY, compactionStrategy.toString()); - } - if (replicaFactor != null) { - options.put(CassandraAdmin.REPLICATION_FACTOR, replicaFactor); - } - if (ru != null) { - options.put(DynamoAdmin.REQUEST_UNIT, ru); - } - if (noScaling != null) { - options.put(DynamoAdmin.NO_SCALING, noScaling.toString()); - } - if (noBackup != null) { - options.put(DynamoAdmin.NO_BACKUP, noBackup.toString()); - } - + Map options = prepareAllOptions(); SchemaLoader.load(configPath, schemaFile, options, coordinator); } @@ -146,10 +127,7 @@ private void repairTables() throws SchemaLoaderException { throw new IllegalArgumentException( "Specifying the '--schema-file' option is required when using the '--repair-all' option"); } - Map options = new HashMap<>(); - if (noBackup != null) { - options.put(DynamoAdmin.NO_BACKUP, noBackup.toString()); - } + Map options = prepareAllOptions(); SchemaLoader.repairTables(configPath, schemaFile, options, coordinator); } @@ -158,10 +136,7 @@ private void alterTables() throws SchemaLoaderException { throw new IllegalArgumentException( "Specifying the '--schema-file' option is required when using the '--alter' option"); } - Map options = new HashMap<>(); - if (noScaling != null) { - options.put(DynamoAdmin.NO_SCALING, noScaling.toString()); - } + Map options = prepareOptions(DynamoAdmin.NO_SCALING); SchemaLoader.alterTables(configPath, schemaFile, options); } @@ -176,7 +151,58 @@ private void importTables() throws SchemaLoaderException { "Specifying the '--coordinator' option with the '--import' option is not allowed." + " Create coordinator tables separately"); } + Map options = prepareAllOptions(); + SchemaLoader.importTables(configPath, schemaFile, options); + } - SchemaLoader.importTables(configPath, schemaFile); + private Map prepareAllOptions() { + return prepareOptions( + CassandraAdmin.REPLICATION_STRATEGY, + CassandraAdmin.COMPACTION_STRATEGY, + CassandraAdmin.REPLICATION_FACTOR, + DynamoAdmin.REQUEST_UNIT, + DynamoAdmin.NO_SCALING, + DynamoAdmin.NO_BACKUP); + } + + private Map prepareOptions(String... options) { + ImmutableMap.Builder optionToValue = ImmutableMap.builder(); + for (String option : options) { + switch (option) { + case CassandraAdmin.REPLICATION_STRATEGY: + if (replicationStrategy != null) { + optionToValue.put(CassandraAdmin.REPLICATION_STRATEGY, replicationStrategy.toString()); + } + break; + case CassandraAdmin.COMPACTION_STRATEGY: + if (compactionStrategy != null) { + optionToValue.put(CassandraAdmin.COMPACTION_STRATEGY, compactionStrategy.toString()); + } + break; + case CassandraAdmin.REPLICATION_FACTOR: + if (replicationFactor != null) { + optionToValue.put(CassandraAdmin.REPLICATION_FACTOR, replicationFactor); + } + break; + case DynamoAdmin.REQUEST_UNIT: + if (ru != null) { + optionToValue.put(DynamoAdmin.REQUEST_UNIT, ru); + } + break; + case DynamoAdmin.NO_SCALING: + if (noScaling != null) { + optionToValue.put(DynamoAdmin.NO_SCALING, noScaling.toString()); + } + break; + case DynamoAdmin.NO_BACKUP: + if (noBackup != null) { + optionToValue.put(DynamoAdmin.NO_BACKUP, noBackup.toString()); + } + break; + default: + throw new AssertionError("Unknown option " + option); + } + } + return optionToValue.build(); } } diff --git a/schema-loader/src/test/java/com/scalar/db/schemaloader/ImportSchemaParserTest.java b/schema-loader/src/test/java/com/scalar/db/schemaloader/ImportSchemaParserTest.java index b00df1c971..9275e22a2a 100644 --- a/schema-loader/src/test/java/com/scalar/db/schemaloader/ImportSchemaParserTest.java +++ b/schema-loader/src/test/java/com/scalar/db/schemaloader/ImportSchemaParserTest.java @@ -1,8 +1,11 @@ package com.scalar.db.schemaloader; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import com.google.common.collect.ImmutableMap; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; public class ImportSchemaParserTest { @@ -11,6 +14,7 @@ public class ImportSchemaParserTest { public void parse_ProperSerializedSchemaJsonAndOptionsGiven_ShouldParseCorrectly() throws SchemaLoaderException { // Arrange + Map globalOptions = ImmutableMap.of("ru", "4000", "replication-factor", "1"); String serializedSchemaJson = "{" + " \"sample_db.sample_table1\": {" @@ -45,7 +49,7 @@ public void parse_ProperSerializedSchemaJsonAndOptionsGiven_ShouldParseCorrectly + " \"compaction-strategy\": \"LCS\"" + " }" + "}"; - ImportSchemaParser parser = new ImportSchemaParser(serializedSchemaJson); + ImportSchemaParser parser = new ImportSchemaParser(serializedSchemaJson, globalOptions); // Act List actual = parser.parse(); @@ -56,13 +60,22 @@ public void parse_ProperSerializedSchemaJsonAndOptionsGiven_ShouldParseCorrectly assertThat(actual.get(0).getNamespace()).isEqualTo("sample_db"); assertThat(actual.get(0).getTable()).isEqualTo("sample_table1"); assertThat(actual.get(0).isTransactionTable()).isTrue(); + assertThat(actual.get(0).getOptions()) + .containsOnly(entry("ru", "4000"), entry("replication-factor", "1")); assertThat(actual.get(1).getNamespace()).isEqualTo("sample_db"); assertThat(actual.get(1).getTable()).isEqualTo("sample_table2"); assertThat(actual.get(1).isTransactionTable()).isFalse(); + assertThat(actual.get(1).getOptions()) + .containsOnly(entry("ru", "4000"), entry("replication-factor", "1")); assertThat(actual.get(2).getNamespace()).isEqualTo("sample_db"); assertThat(actual.get(2).getTable()).isEqualTo("sample_table3"); assertThat(actual.get(2).isTransactionTable()).isTrue(); + assertThat(actual.get(2).getOptions()) + .containsOnly( + entry("ru", "5000"), + entry("compaction-strategy", "LCS"), + entry("replication-factor", "1")); } } diff --git a/schema-loader/src/test/java/com/scalar/db/schemaloader/ImportTableSchemaTest.java b/schema-loader/src/test/java/com/scalar/db/schemaloader/ImportTableSchemaTest.java index 1ad062e9c1..fd366009ac 100644 --- a/schema-loader/src/test/java/com/scalar/db/schemaloader/ImportTableSchemaTest.java +++ b/schema-loader/src/test/java/com/scalar/db/schemaloader/ImportTableSchemaTest.java @@ -1,7 +1,11 @@ package com.scalar.db.schemaloader; +import static org.assertj.core.api.Assertions.entry; + +import com.google.common.collect.ImmutableMap; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import java.util.Collections; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -24,12 +28,14 @@ public void constructor_DefinitionWithTransactionTrueGiven_ShouldConstructProper JsonObject tableDefinition = JsonParser.parseString(tableDefinitionJson).getAsJsonObject(); // Act - ImportTableSchema tableSchema = new ImportTableSchema("ns.tbl", tableDefinition); + ImportTableSchema tableSchema = + new ImportTableSchema("ns.tbl", tableDefinition, Collections.emptyMap()); // Assert Assertions.assertThat(tableSchema.getNamespace()).isEqualTo("ns"); Assertions.assertThat(tableSchema.getTable()).isEqualTo("tbl"); Assertions.assertThat(tableSchema.isTransactionTable()).isEqualTo(true); + Assertions.assertThat(tableSchema.getOptions()).isEmpty(); } @Test @@ -39,12 +45,14 @@ public void constructor_DefinitionWithTransactionFalseGiven_ShouldConstructPrope JsonObject tableDefinition = JsonParser.parseString(tableDefinitionJson).getAsJsonObject(); // Act - ImportTableSchema tableSchema = new ImportTableSchema("ns.tbl", tableDefinition); + ImportTableSchema tableSchema = + new ImportTableSchema("ns.tbl", tableDefinition, Collections.emptyMap()); // Assert Assertions.assertThat(tableSchema.getNamespace()).isEqualTo("ns"); Assertions.assertThat(tableSchema.getTable()).isEqualTo("tbl"); Assertions.assertThat(tableSchema.isTransactionTable()).isEqualTo(false); + Assertions.assertThat(tableSchema.getOptions()).isEmpty(); } @Test @@ -53,7 +61,8 @@ public void constructor_WrongFormatTableFullNameGiven_ShouldThrowSchemaLoaderExc String tableFullName = "namespace_and_table_without_dot_separator"; // Act Assert - Assertions.assertThatThrownBy(() -> new ImportTableSchema(tableFullName, tableDefinition)) + Assertions.assertThatThrownBy( + () -> new ImportTableSchema(tableFullName, tableDefinition, Collections.emptyMap())) .isInstanceOf(SchemaLoaderException.class); } @@ -64,11 +73,38 @@ public void constructor_DefinitionWithoutTransactionGiven_ShouldConstructProperT JsonObject tableDefinition = JsonParser.parseString(tableDefinitionJson).getAsJsonObject(); // Act - ImportTableSchema tableSchema = new ImportTableSchema("ns.tbl", tableDefinition); + ImportTableSchema tableSchema = + new ImportTableSchema("ns.tbl", tableDefinition, Collections.emptyMap()); // Assert Assertions.assertThat(tableSchema.getNamespace()).isEqualTo("ns"); Assertions.assertThat(tableSchema.getTable()).isEqualTo("tbl"); Assertions.assertThat(tableSchema.isTransactionTable()).isEqualTo(true); + Assertions.assertThat(tableSchema.getOptions()).isEmpty(); + } + + @Test + public void constructor_DefinitionWithGlobalAndSchemaOptions_ShouldConstructWithProperOptions() + throws SchemaLoaderException { + String tableDefinitionJson = + "{\"partition-key\": \"ignored\", \"columns\": \"ignored\", \"clustering-key\": \"ignored\", \"secondary-index\": \"ignored\",\"transaction\": false, \"opt1\": \"schema-opt1\", \"opt3\": \"schema-opt3\"}"; + JsonObject tableDefinition = JsonParser.parseString(tableDefinitionJson).getAsJsonObject(); + + // Act + ImportTableSchema tableSchema = + new ImportTableSchema( + "ns.tbl", + tableDefinition, + ImmutableMap.of("opt1", "global-opt1", "opt2", "global-opt2")); + + // Assert + Assertions.assertThat(tableSchema.getNamespace()).isEqualTo("ns"); + Assertions.assertThat(tableSchema.getTable()).isEqualTo("tbl"); + Assertions.assertThat(tableSchema.isTransactionTable()).isEqualTo(false); + Assertions.assertThat(tableSchema.getOptions()) + .containsOnly( + entry("opt1", "schema-opt1"), + entry("opt2", "global-opt2"), + entry("opt3", "schema-opt3")); } } diff --git a/schema-loader/src/test/java/com/scalar/db/schemaloader/SchemaLoaderTest.java b/schema-loader/src/test/java/com/scalar/db/schemaloader/SchemaLoaderTest.java index e887e532e2..a8b86e643f 100644 --- a/schema-loader/src/test/java/com/scalar/db/schemaloader/SchemaLoaderTest.java +++ b/schema-loader/src/test/java/com/scalar/db/schemaloader/SchemaLoaderTest.java @@ -2,6 +2,7 @@ import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mockStatic; @@ -46,7 +47,7 @@ public void setUp() throws SchemaLoaderException { .when(() -> SchemaLoader.getSchemaParser(any(), anyMap())) .thenReturn(parser); schemaLoaderMockedStatic - .when(() -> SchemaLoader.getImportSchemaParser(any())) + .when(() -> SchemaLoader.getImportSchemaParser(any(), anyMap())) .thenReturn(importSchemaParser); when(parser.parse()).thenReturn(Collections.emptyList()); when(importSchemaParser.parse()).thenReturn(Collections.emptyList()); @@ -719,11 +720,11 @@ public void alterTable_WithConfigPropertiesAndSchemaFilePath_ShouldCallParserAnd // Arrange // Act - SchemaLoader.importTables(configProperties, SERIALIZED_SCHEMA_JSON); + SchemaLoader.importTables(configProperties, SERIALIZED_SCHEMA_JSON, options); // Assert verify(importSchemaParser).parse(); - verify(operator).importTables(anyList()); + verify(operator).importTables(anyList(), eq(options)); } @Test @@ -733,11 +734,11 @@ public void alterTable_WithConfigPropertiesAndSchemaFilePath_ShouldCallParserAnd // Arrange // Act - SchemaLoader.importTables(configFilePath, SERIALIZED_SCHEMA_JSON); + SchemaLoader.importTables(configFilePath, SERIALIZED_SCHEMA_JSON, options); // Assert verify(importSchemaParser).parse(); - verify(operator).importTables(anyList()); + verify(operator).importTables(anyList(), eq(options)); } @Test @@ -747,11 +748,11 @@ public void alterTable_WithConfigPropertiesAndSchemaFilePath_ShouldCallParserAnd // Arrange // Act - SchemaLoader.importTables(configProperties, schemaFilePath); + SchemaLoader.importTables(configProperties, schemaFilePath, options); // Assert verify(importSchemaParser).parse(); - verify(operator).importTables(anyList()); + verify(operator).importTables(anyList(), eq(options)); } @Test @@ -760,10 +761,10 @@ public void importTable_WithConfigFilePathAndSchemaFilePath_ShouldCallParserAndO // Arrange // Act - SchemaLoader.importTables(configFilePath, schemaFilePath); + SchemaLoader.importTables(configFilePath, schemaFilePath, options); // Assert verify(importSchemaParser).parse(); - verify(operator).importTables(anyList()); + verify(operator).importTables(anyList(), eq(options)); } } diff --git a/schema-loader/src/test/java/com/scalar/db/schemaloader/SchemaOperatorTest.java b/schema-loader/src/test/java/com/scalar/db/schemaloader/SchemaOperatorTest.java index e707282118..9b310e9666 100644 --- a/schema-loader/src/test/java/com/scalar/db/schemaloader/SchemaOperatorTest.java +++ b/schema-loader/src/test/java/com/scalar/db/schemaloader/SchemaOperatorTest.java @@ -513,10 +513,10 @@ public void importTables_WithTransactionTables_ShouldCallProperMethods() throws when(importTableSchema.getTable()).thenReturn("tb"); // Act - operator.importTables(tableSchemaList); + operator.importTables(tableSchemaList, options); // Assert - verify(transactionAdmin, times(3)).importTable("ns", "tb"); + verify(transactionAdmin, times(3)).importTable("ns", "tb", options); verifyNoInteractions(storageAdmin); } @@ -530,10 +530,10 @@ public void importTables_WithoutTransactionTables_ShouldCallProperMethods() thro when(importTableSchema.getTable()).thenReturn("tb"); // Act - operator.importTables(tableSchemaList); + operator.importTables(tableSchemaList, options); // Assert - verify(storageAdmin, times(3)).importTable("ns", "tb"); + verify(storageAdmin, times(3)).importTable("ns", "tb", options); verifyNoInteractions(transactionAdmin); } } diff --git a/schema-loader/src/test/java/com/scalar/db/schemaloader/command/SchemaLoaderCommandTest.java b/schema-loader/src/test/java/com/scalar/db/schemaloader/command/SchemaLoaderCommandTest.java index 56d21771a1..28911a0e1e 100644 --- a/schema-loader/src/test/java/com/scalar/db/schemaloader/command/SchemaLoaderCommandTest.java +++ b/schema-loader/src/test/java/com/scalar/db/schemaloader/command/SchemaLoaderCommandTest.java @@ -355,13 +355,37 @@ public void call_ImportOptionGivenWithProperArguments_ShouldCallRepairTablePrope // Arrange String schemaFile = "path_to_file"; String configFile = "path_to_config_file"; + Map options = + ImmutableMap.builder() + .put(CassandraAdmin.REPLICATION_STRATEGY, replicationStrategy) + .put(CassandraAdmin.COMPACTION_STRATEGY, compactionStrategy) + .put(CassandraAdmin.REPLICATION_FACTOR, replicationFactor) + .put(DynamoAdmin.REQUEST_UNIT, ru) + .put(DynamoAdmin.NO_SCALING, noScaling.toString()) + .put(DynamoAdmin.NO_BACKUP, noBackup.toString()) + .build(); // Act - commandLine.execute("-f", schemaFile, "--import", "--config", configFile); + commandLine.execute( + "-f", + schemaFile, + "--import", + "--config", + configFile, + "--replication-strategy", + replicationStrategy, + "--compaction-strategy", + compactionStrategy, + "--replication-factor", + replicationFactor, + "--ru", + ru, + "--no-scaling", + "--no-backup"); // Assert schemaLoaderMockedStatic.verify( - () -> SchemaLoader.importTables(Paths.get(configFile), Paths.get(schemaFile))); + () -> SchemaLoader.importTables(Paths.get(configFile), Paths.get(schemaFile), options)); } @Test