diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index c3b64554776b..90b326036544 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -153,7 +153,7 @@ public class HiveIcebergMetaHook implements HiveMetaHook { AlterTableType.ADDPROPS, AlterTableType.DROPPROPS, AlterTableType.SETPARTITIONSPEC, AlterTableType.UPDATE_COLUMNS, AlterTableType.RENAME, AlterTableType.EXECUTE, AlterTableType.CREATE_BRANCH, AlterTableType.CREATE_TAG, AlterTableType.DROP_BRANCH, AlterTableType.RENAME_BRANCH, AlterTableType.DROPPARTITION, - AlterTableType.DROP_TAG, AlterTableType.COMPACT); + AlterTableType.DROP_TAG, AlterTableType.COMPACT, AlterTableType.REPLACE_BRANCH); private static final List MIGRATION_ALLOWED_SOURCE_FORMATS = ImmutableList.of( FileFormat.PARQUET.name().toLowerCase(), FileFormat.ORC.name().toLowerCase(), diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 369bc91fbcc4..cf59e18685ad 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -1008,6 +1008,11 @@ public void alterTableSnapshotRefOperation(org.apache.hadoop.hive.ql.metadata.Ta (AlterTableSnapshotRefSpec.RenameSnapshotrefSpec) alterTableSnapshotRefSpec.getOperationParams(); IcebergBranchExec.renameBranch(icebergTable, renameSnapshotrefSpec); break; + case REPLACE_BRANCH: + AlterTableSnapshotRefSpec.ReplaceSnapshotrefSpec replaceSnapshotrefSpec = + (AlterTableSnapshotRefSpec.ReplaceSnapshotrefSpec) alterTableSnapshotRefSpec.getOperationParams(); + IcebergBranchExec.replaceBranch(icebergTable, replaceSnapshotrefSpec); + break; case DROP_TAG: AlterTableSnapshotRefSpec.DropSnapshotRefSpec dropTagSpec = (AlterTableSnapshotRefSpec.DropSnapshotRefSpec) alterTableSnapshotRefSpec.getOperationParams(); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java index 9370d7252438..8f54aef4aa09 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java @@ -97,4 +97,29 @@ public static void renameBranch(Table table, AlterTableSnapshotRefSpec.RenameSna LOG.info("Renaming branch {} to {} on iceberg table {}", sourceBranch, targetBranch, table.name()); table.manageSnapshots().renameBranch(sourceBranch, targetBranch).commit(); } + + public static void replaceBranch(Table table, + AlterTableSnapshotRefSpec.ReplaceSnapshotrefSpec replaceSnapshotrefSpec) { + String sourceBranch = replaceSnapshotrefSpec.getSourceBranchName(); + ManageSnapshots manageSnapshots; + if (replaceSnapshotrefSpec.isReplaceBySnapshot()) { + long targetSnapshot = replaceSnapshotrefSpec.getTargetSnapshot(); + LOG.info("Replacing branch {} with snapshot {} on iceberg table {}", sourceBranch, targetSnapshot, table.name()); + manageSnapshots = table.manageSnapshots().replaceBranch(sourceBranch, targetSnapshot); + } else { + String targetBranch = replaceSnapshotrefSpec.getTargetBranchName(); + LOG.info("Replacing branch {} with branch {} on iceberg table {}", sourceBranch, targetBranch, table.name()); + manageSnapshots = table.manageSnapshots().replaceBranch(sourceBranch, targetBranch); + } + if (replaceSnapshotrefSpec.getMaxRefAgeMs() > 0) { + manageSnapshots.setMaxRefAgeMs(sourceBranch, replaceSnapshotrefSpec.getMaxRefAgeMs()); + } + if (replaceSnapshotrefSpec.getMaxSnapshotAgeMs() > 0) { + manageSnapshots.setMaxSnapshotAgeMs(sourceBranch, replaceSnapshotrefSpec.getMaxSnapshotAgeMs()); + } + if (replaceSnapshotrefSpec.getMinSnapshotsToKeep() > 0) { + manageSnapshots.setMinSnapshotsToKeep(sourceBranch, replaceSnapshotrefSpec.getMinSnapshotsToKeep()); + } + manageSnapshots.commit(); + } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCherryPick.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSnapshotOperations.java similarity index 64% rename from iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCherryPick.java rename to iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSnapshotOperations.java index 99e2c6d6e2cf..7f92fcea4932 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCherryPick.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSnapshotOperations.java @@ -32,7 +32,7 @@ import static org.apache.iceberg.mr.hive.TestTables.TestTableType.HIVE_CATALOG; import static org.junit.Assert.assertEquals; -public class TestHiveIcebergCherryPick { +public class TestHiveIcebergSnapshotOperations { private TestTables testTables; private TestHiveShell shell; @@ -82,4 +82,38 @@ public void testCherryPick() { List result = shell.executeStatement("SELECT COUNT(*) FROM " + identifier.name()); assertEquals(6L, result.get(0)[0]); } + + @Test + public void testReplaceBranchWithSnapshot() { + TableIdentifier identifier = TableIdentifier.of("default", "testReplaceBranchWithSnapshot"); + shell.executeStatement( + String.format("CREATE EXTERNAL TABLE %s (id INT) STORED BY iceberg %s %s", + identifier.name(), + testTables.locationForCreateTableSQL(identifier), + testTables.propertiesForCreateTableSQL(ImmutableMap.of()))); + + shell.executeStatement(String.format("INSERT INTO TABLE %s VALUES(1),(2),(3),(4)", identifier.name())); + + org.apache.iceberg.Table icebergTable = testTables.loadTable(identifier); + icebergTable.refresh(); + // Create a branch + shell.executeStatement(String.format("ALTER TABLE %s create branch branch1", identifier.name())); + // Make one new insert to the main branch + shell.executeStatement(String.format("INSERT INTO TABLE %s VALUES(5),(6)", identifier.name())); + icebergTable.refresh(); + long id = icebergTable.currentSnapshot().snapshotId(); + + // Make another insert so that the commit isn't the last commit on the branch + shell.executeStatement(String.format("INSERT INTO TABLE %s VALUES(7),(8)", identifier.name())); + + // Validate the original count on branch before replace + List result = + shell.executeStatement("SELECT COUNT(*) FROM default.testReplaceBranchWithSnapshot.branch_branch1"); + assertEquals(4L, result.get(0)[0]); + // Perform replace branch with snapshot id. + shell.executeStatement( + String.format("ALTER TABLE %s replace branch branch1 as of system_version %s", identifier.name(), id)); + result = shell.executeStatement("SELECT COUNT(*) FROM default.testReplaceBranchWithSnapshot.branch_branch1"); + assertEquals(6L, result.get(0)[0]); + } } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/replace_iceberg_branch.q b/iceberg/iceberg-handler/src/test/queries/positive/replace_iceberg_branch.q new file mode 100644 index 000000000000..0186db25e9e4 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/replace_iceberg_branch.q @@ -0,0 +1,46 @@ +-- SORT_QUERY_RESULTS +set hive.explain.user=false; +set hive.fetch.task.conversion=more; + +create external table ice01(id int) stored by iceberg stored as orc tblproperties ('format-version'='2'); + +insert into ice01 values (1), (2), (3), (4); + +select * from ice01; + +-- create one branch +alter table ice01 create branch branch1; + +-- insert some values in branch1 +insert into default.ice01.branch_branch1 values (5), (6); +select * from default.ice01.branch_branch1; + +-- create another branch +alter table ice01 create branch branch2; +-- do some inserts & deletes on this branch +insert into default.ice01.branch_branch2 values (22), (44); +delete from default.ice01.branch_branch2 where id=2; +select * from default.ice01.branch_branch2; + +-- Do a replace +explain alter table ice01 replace branch branch1 as of branch branch2; +alter table ice01 replace branch branch1 as of branch branch2; +select * from default.ice01.branch_branch1; + +-- create another branch +alter table ice01 create branch branch3; +-- do some inserts & deletes on this branch +insert into default.ice01.branch_branch3 values (45), (32); + +-- Do a replace with retain last +explain alter table ice01 replace branch branch1 as of branch branch3 retain 5 days; +alter table ice01 replace branch branch1 as of branch branch3 retain 5 days; +select * from default.ice01.branch_branch1; + +-- create another branch +alter table ice01 create branch branch4; +-- do some inserts & deletes on this branch +insert into default.ice01.branch_branch4 values (11), (78); +explain alter table ice01 replace branch branch1 as of branch branch4 with snapshot retention 5 snapshots 6 days; +alter table ice01 replace branch branch1 as of branch branch4 with snapshot retention 5 snapshots 6 days; +select * from default.ice01.branch_branch1; diff --git a/iceberg/iceberg-handler/src/test/results/positive/replace_iceberg_branch.q.out b/iceberg/iceberg-handler/src/test/results/positive/replace_iceberg_branch.q.out new file mode 100644 index 000000000000..1d964903e221 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/replace_iceberg_branch.q.out @@ -0,0 +1,223 @@ +PREHOOK: query: create external table ice01(id int) stored by iceberg stored as orc tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice01 +POSTHOOK: query: create external table ice01(id int) stored by iceberg stored as orc tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice01 +PREHOOK: query: insert into ice01 values (1), (2), (3), (4) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice01 +POSTHOOK: query: insert into ice01 values (1), (2), (3), (4) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice01 +PREHOOK: query: select * from ice01 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from ice01 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +2 +3 +4 +PREHOOK: query: alter table ice01 create branch branch1 +PREHOOK: type: ALTERTABLE_CREATEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: alter table ice01 create branch branch1 +POSTHOOK: type: ALTERTABLE_CREATEBRANCH +POSTHOOK: Input: default@ice01 +PREHOOK: query: insert into default.ice01.branch_branch1 values (5), (6) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice01 +POSTHOOK: query: insert into default.ice01.branch_branch1 values (5), (6) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice01 +PREHOOK: query: select * from default.ice01.branch_branch1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from default.ice01.branch_branch1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +2 +3 +4 +5 +6 +PREHOOK: query: alter table ice01 create branch branch2 +PREHOOK: type: ALTERTABLE_CREATEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: alter table ice01 create branch branch2 +POSTHOOK: type: ALTERTABLE_CREATEBRANCH +POSTHOOK: Input: default@ice01 +PREHOOK: query: insert into default.ice01.branch_branch2 values (22), (44) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice01 +POSTHOOK: query: insert into default.ice01.branch_branch2 values (22), (44) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice01 +PREHOOK: query: delete from default.ice01.branch_branch2 where id=2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: default@ice01 +POSTHOOK: query: delete from default.ice01.branch_branch2 where id=2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: default@ice01 +PREHOOK: query: select * from default.ice01.branch_branch2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from default.ice01.branch_branch2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +22 +3 +4 +44 +PREHOOK: query: explain alter table ice01 replace branch branch1 as of branch branch2 +PREHOOK: type: ALTERTABLE_REPLACEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: explain alter table ice01 replace branch branch1 as of branch branch2 +POSTHOOK: type: ALTERTABLE_REPLACEBRANCH +POSTHOOK: Input: default@ice01 +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + SnapshotRef Operation + table name: default.ice01 + spec: AlterTableSnapshotRefSpec{operationType=REPLACE_BRANCH, operationParams=ReplaceSnapshotrefSpec{sourceBranch=branch1, targetBranch=branch2}} + +PREHOOK: query: alter table ice01 replace branch branch1 as of branch branch2 +PREHOOK: type: ALTERTABLE_REPLACEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: alter table ice01 replace branch branch1 as of branch branch2 +POSTHOOK: type: ALTERTABLE_REPLACEBRANCH +POSTHOOK: Input: default@ice01 +PREHOOK: query: select * from default.ice01.branch_branch1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from default.ice01.branch_branch1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +22 +3 +4 +44 +PREHOOK: query: alter table ice01 create branch branch3 +PREHOOK: type: ALTERTABLE_CREATEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: alter table ice01 create branch branch3 +POSTHOOK: type: ALTERTABLE_CREATEBRANCH +POSTHOOK: Input: default@ice01 +PREHOOK: query: insert into default.ice01.branch_branch3 values (45), (32) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice01 +POSTHOOK: query: insert into default.ice01.branch_branch3 values (45), (32) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice01 +PREHOOK: query: explain alter table ice01 replace branch branch1 as of branch branch3 retain 5 days +PREHOOK: type: ALTERTABLE_REPLACEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: explain alter table ice01 replace branch branch1 as of branch branch3 retain 5 days +POSTHOOK: type: ALTERTABLE_REPLACEBRANCH +POSTHOOK: Input: default@ice01 +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + SnapshotRef Operation + table name: default.ice01 + spec: AlterTableSnapshotRefSpec{operationType=REPLACE_BRANCH, operationParams=ReplaceSnapshotrefSpec{sourceBranch=branch1, targetBranch=branch3, maxRefAgeMs=432000000}} + +PREHOOK: query: alter table ice01 replace branch branch1 as of branch branch3 retain 5 days +PREHOOK: type: ALTERTABLE_REPLACEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: alter table ice01 replace branch branch1 as of branch branch3 retain 5 days +POSTHOOK: type: ALTERTABLE_REPLACEBRANCH +POSTHOOK: Input: default@ice01 +PREHOOK: query: select * from default.ice01.branch_branch1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from default.ice01.branch_branch1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +2 +3 +32 +4 +45 +PREHOOK: query: alter table ice01 create branch branch4 +PREHOOK: type: ALTERTABLE_CREATEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: alter table ice01 create branch branch4 +POSTHOOK: type: ALTERTABLE_CREATEBRANCH +POSTHOOK: Input: default@ice01 +PREHOOK: query: insert into default.ice01.branch_branch4 values (11), (78) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice01 +POSTHOOK: query: insert into default.ice01.branch_branch4 values (11), (78) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice01 +PREHOOK: query: explain alter table ice01 replace branch branch1 as of branch branch4 with snapshot retention 5 snapshots 6 days +PREHOOK: type: ALTERTABLE_REPLACEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: explain alter table ice01 replace branch branch1 as of branch branch4 with snapshot retention 5 snapshots 6 days +POSTHOOK: type: ALTERTABLE_REPLACEBRANCH +POSTHOOK: Input: default@ice01 +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + SnapshotRef Operation + table name: default.ice01 + spec: AlterTableSnapshotRefSpec{operationType=REPLACE_BRANCH, operationParams=ReplaceSnapshotrefSpec{sourceBranch=branch1, targetBranch=branch4, minSnapshotsToKeep=5, maxSnapshotAgeMs=518400000}} + +PREHOOK: query: alter table ice01 replace branch branch1 as of branch branch4 with snapshot retention 5 snapshots 6 days +PREHOOK: type: ALTERTABLE_REPLACEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: alter table ice01 replace branch branch1 as of branch branch4 with snapshot retention 5 snapshots 6 days +POSTHOOK: type: ALTERTABLE_REPLACEBRANCH +POSTHOOK: Input: default@ice01 +PREHOOK: query: select * from default.ice01.branch_branch1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from default.ice01.branch_branch1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +11 +2 +3 +4 +78 diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g index a628c4364ed5..c6bbcfca3859 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g @@ -81,6 +81,7 @@ alterTableStatementSuffix | alterStatementSuffixDropTag | alterStatementSuffixConvert | alterStatementSuffixRenameBranch + | alterStatementSuffixReplaceBranch ; alterTblPartitionStatementSuffix[boolean partition] @@ -513,6 +514,13 @@ alterStatementSuffixRenameBranch -> ^(TOK_ALTERTABLE_RENAME_BRANCH $sourceBranch $targetBranch) ; +alterStatementSuffixReplaceBranch +@init { gParent.pushMsg("alter table replace branch", state); } +@after { gParent.popMsg(state); } + : KW_REPLACE KW_BRANCH sourceBranch=Identifier KW_AS KW_OF ((KW_SYSTEM_VERSION snapshotId=Number) | (KW_BRANCH branch=identifier)) refRetain? retentionOfSnapshots? + -> ^(TOK_ALTERTABLE_REPLACE_BRANCH $sourceBranch KW_SYSTEM_VERSION? $snapshotId? $branch? refRetain? retentionOfSnapshots?) + ; + alterStatementSuffixDropBranch @init { gParent.pushMsg("alter table drop branch (if exists) branchName", state); } @after { gParent.popMsg(state); } diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 9c2c0c6b3027..f128666d3cf6 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -222,6 +222,7 @@ TOK_ALTERTABLE_EXECUTE; TOK_ALTERTABLE_CREATE_BRANCH; TOK_ALTERTABLE_DROP_BRANCH; TOK_ALTERTABLE_RENAME_BRANCH; +TOK_ALTERTABLE_REPLACE_BRANCH; TOK_ALTERTABLE_CREATE_TAG; TOK_ALTERTABLE_DROP_TAG; TOK_RETAIN; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java index cc78ba6317dc..d0d104e0712c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java @@ -43,6 +43,7 @@ public enum AlterTableType { CREATE_BRANCH("create branch"), DROP_BRANCH("drop branch"), RENAME_BRANCH("rename branch"), + REPLACE_BRANCH("replace branch"), CREATE_TAG("create tag"), DROP_TAG("drop tag"), // constraint diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/replace/AlterTableReplaceBranchRefAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/replace/AlterTableReplaceBranchRefAnalyzer.java new file mode 100644 index 000000000000..06d8aa470cee --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/replace/AlterTableReplaceBranchRefAnalyzer.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.ddl.table.snapshotref.branch.replace; + +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; +import org.apache.hadoop.hive.ql.ddl.DDLWork; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; +import org.apache.hadoop.hive.ql.ddl.table.snapshotref.AlterTableSnapshotRefDesc; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +import static org.apache.hadoop.hive.ql.parse.HiveParser_AlterClauseParser.KW_SYSTEM_VERSION; + +@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERTABLE_REPLACE_BRANCH) +public class AlterTableReplaceBranchRefAnalyzer extends AbstractAlterTableAnalyzer { + + protected AlterTableType alterTableType; + + public AlterTableReplaceBranchRefAnalyzer(QueryState queryState) throws SemanticException { + super(queryState); + alterTableType = AlterTableType.REPLACE_BRANCH; + } + + @Override + protected void analyzeCommand(TableName tableName, Map partitionSpec, ASTNode command) + throws SemanticException { + Table table = getTable(tableName); + DDLUtils.validateTableIsIceberg(table); + inputs.add(new ReadEntity(table)); + validateAlterTableType(table, alterTableType, false); + AlterTableSnapshotRefSpec.ReplaceSnapshotrefSpec replaceSnapshotrefSpec; + String sourceBranch = command.getChild(0).getText(); + int childNodeNum; + if (command.getChild(1).getType() == KW_SYSTEM_VERSION) { + long targetSnapshot = Long.parseLong(command.getChild(2).getText()); + replaceSnapshotrefSpec = new AlterTableSnapshotRefSpec.ReplaceSnapshotrefSpec(sourceBranch, targetSnapshot); + childNodeNum = 3; + } else { + String targetBranch = command.getChild(1).getText(); + replaceSnapshotrefSpec = new AlterTableSnapshotRefSpec.ReplaceSnapshotrefSpec(sourceBranch, targetBranch); + childNodeNum = 2; + } + + for (; childNodeNum < command.getChildCount(); childNodeNum++) { + ASTNode childNode = (ASTNode) command.getChild(childNodeNum); + switch (childNode.getToken().getType()) { + case HiveParser.TOK_RETAIN: + String maxRefAge = childNode.getChild(0).getText(); + String timeUnitOfBranchRetain = childNode.getChild(1).getText(); + long maxRefAgeMs = + TimeUnit.valueOf(timeUnitOfBranchRetain.toUpperCase(Locale.ENGLISH)).toMillis(Long.parseLong(maxRefAge)); + replaceSnapshotrefSpec.setMaxRefAgeMs(maxRefAgeMs); + break; + case HiveParser.TOK_WITH_SNAPSHOT_RETENTION: + int minSnapshotsToKeep = Integer.parseInt(childNode.getChild(0).getText()); + replaceSnapshotrefSpec.setMinSnapshotsToKeep(minSnapshotsToKeep); + if (childNode.getChildren().size() > 1) { + String maxSnapshotAge = childNode.getChild(1).getText(); + String timeUnitOfSnapshotsRetention = childNode.getChild(2).getText(); + long maxSnapshotAgeMs = TimeUnit.valueOf(timeUnitOfSnapshotsRetention.toUpperCase(Locale.ENGLISH)) + .toMillis(Long.parseLong(maxSnapshotAge)); + replaceSnapshotrefSpec.setMaxSnapshotAgeMs(maxSnapshotAgeMs); + } + break; + default: + throw new SemanticException("Unrecognized token in ALTER " + alterTableType.getName() + " statement"); + } + } + + AlterTableSnapshotRefSpec alterTableSnapshotRefSpec = + new AlterTableSnapshotRefSpec<>(alterTableType, replaceSnapshotrefSpec); + AbstractAlterTableDesc alterTableDesc = + new AlterTableSnapshotRefDesc(alterTableType, tableName, alterTableSnapshotRefSpec); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTableDesc))); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java index 004086528300..dfe4c0d1071b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.parse; import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; + import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; public class AlterTableSnapshotRefSpec { @@ -147,4 +149,91 @@ public String toString() { .toString(); } } + + public static class ReplaceSnapshotrefSpec { + + private final String sourceBranch; + private String targetBranch = null; + private long targetSnapshot; + + boolean replaceBySnapshot = false; + private long maxRefAgeMs = -1; + private int minSnapshotsToKeep = -1; + private long maxSnapshotAgeMs = -1; + + public String getSourceBranchName() { + return sourceBranch; + } + + public String getTargetBranchName() { + return targetBranch; + } + + public boolean isReplaceBySnapshot() { + return replaceBySnapshot; + } + + public long getTargetSnapshot() { + return targetSnapshot; + } + + public ReplaceSnapshotrefSpec(String sourceBranch, String targetBranch) { + this.sourceBranch = sourceBranch; + this.targetBranch = targetBranch; + } + + public ReplaceSnapshotrefSpec(String sourceBranch, long targetSnapshot) { + this.sourceBranch = sourceBranch; + this.targetSnapshot = targetSnapshot; + replaceBySnapshot = true; + } + + public void setMaxRefAgeMs(long maxRefAgeMs) { + Preconditions.checkArgument(maxRefAgeMs > 0); + this.maxRefAgeMs = maxRefAgeMs; + } + + public void setMinSnapshotsToKeep(int minSnapshotsToKeep) { + Preconditions.checkArgument(minSnapshotsToKeep > 0); + this.minSnapshotsToKeep = minSnapshotsToKeep; + } + + public void setMaxSnapshotAgeMs(long maxSnapshotAgeMs) { + Preconditions.checkArgument(maxSnapshotAgeMs > 0); + this.maxSnapshotAgeMs = maxSnapshotAgeMs; + } + + public long getMaxRefAgeMs() { + return maxRefAgeMs; + } + + public int getMinSnapshotsToKeep() { + return minSnapshotsToKeep; + } + + public long getMaxSnapshotAgeMs() { + return maxSnapshotAgeMs; + } + + @Override + public String toString() { + MoreObjects.ToStringHelper stringHelper = MoreObjects.toStringHelper(this); + stringHelper.add("sourceBranch", sourceBranch); + if (replaceBySnapshot) { + stringHelper.add("targetSnapshot", targetSnapshot); + } else { + stringHelper.add("targetBranch", targetBranch); + } + if (maxRefAgeMs != -1) { + stringHelper.add("maxRefAgeMs", maxRefAgeMs); + } + if (minSnapshotsToKeep != -1) { + stringHelper.add("minSnapshotsToKeep", minSnapshotsToKeep); + } + if (maxSnapshotAgeMs != -1) { + stringHelper.add("maxSnapshotAgeMs", maxSnapshotAgeMs); + } + return stringHelper.toString(); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java index d616912659a3..edfa71da1c85 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java @@ -81,6 +81,7 @@ public enum HiveOperation { ALTERTABLE_CREATETAG("ALTERTABLE_CREATETAG", HiveParser.TOK_ALTERTABLE_CREATE_TAG, null, null), ALTERTABLE_DROPBRANCH("ALTERTABLE_DROPBRANCH", HiveParser.TOK_ALTERTABLE_DROP_BRANCH, null, null), ALTERTABLE_RENAMEBRANCH("ALTERTABLE_RENAMEBRANCH", HiveParser.TOK_ALTERTABLE_RENAME_BRANCH, null, null), + ALTERTABLE_REPLACEBRANCH("ALTERTABLE_REPLACEBRANCH", HiveParser.TOK_ALTERTABLE_REPLACE_BRANCH, null, null), ALTERTABLE_DROPTAG("ALTERTABLE_DROPTAG", HiveParser.TOK_ALTERTABLE_DROP_TAG, null, null), ALTERTABLE_CONVERT("ALTERTABLE_CONVERT", HiveParser.TOK_ALTERTABLE_CONVERT, null, null), ALTERTABLE_SERIALIZER("ALTERTABLE_SERIALIZER", HiveParser.TOK_ALTERTABLE_SERIALIZER, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java index e09cc8c78084..bc1b40583dad 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java @@ -142,6 +142,7 @@ public enum HiveOperationType { ALTERTABLE_CREATEBRANCH, ALTERTABLE_DROPBRANCH, ALTERTABLE_RENAMEBRANCH, + ALTERTABLE_REPLACEBRANCH, ALTERTABLE_CREATETAG, ALTERTABLE_DROPTAG, SHOW_COMPACTIONS, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java index b200522f5a9d..191e3a3f93f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java @@ -248,6 +248,8 @@ public HivePrivilegeObjectType getObjectType() { PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); op2Priv.put(HiveOperationType.ALTERTABLE_RENAMEBRANCH, PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); + op2Priv.put(HiveOperationType.ALTERTABLE_REPLACEBRANCH, + PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); op2Priv.put(HiveOperationType.ALTERTABLE_DROPTAG, PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR));