From 5b50628507b4fbcf18b86e59c6f23fc8e8c8e02c Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Tue, 19 Nov 2024 10:09:17 +0800 Subject: [PATCH] [BugFix] Ignore to refresh if base table has dropped one partition (#52740) Signed-off-by: shuming.li --- .../partitiontraits/DefaultTraits.java | 4 ++-- .../partitiontraits/OlapPartitionTraits.java | 4 ++-- ...titionBasedMvRefreshProcessorHiveTest.java | 22 ++++--------------- 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/DefaultTraits.java b/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/DefaultTraits.java index 5ca2f4e9984e8..1284efc1694e3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/DefaultTraits.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/DefaultTraits.java @@ -161,8 +161,8 @@ public Set getUpdatedPartitionNames(List baseTables, for (Map.Entry versionEntry : versionMap.entrySet()) { String basePartitionName = versionEntry.getKey(); if (!latestPartitionInfo.containsKey(basePartitionName)) { - // TODO: If one partition has been dropped, refresh all existed partitions again. - return latestPartitionInfo.keySet(); + // If this partition is dropped, ignore it. + continue; } long basePartitionVersion = latestPartitionInfo.get(basePartitionName).getModifiedTime(); diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/OlapPartitionTraits.java b/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/OlapPartitionTraits.java index 52d58adfad510..76036caffed56 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/OlapPartitionTraits.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/OlapPartitionTraits.java @@ -106,8 +106,8 @@ public Set getUpdatedPartitionNames(List baseTables, String basePartitionName = versionEntry.getKey(); Partition basePartition = baseTable.getPartition(basePartitionName); if (basePartition == null) { - // TODO: Once there is a partition deleted, refresh all partitions. - return baseTable.getVisiblePartitionNames(); + // If this partition is dropped, ignore it. + continue; } MaterializedView.BasePartitionInfo mvRefreshedPartitionInfo = versionEntry.getValue(); if (mvRefreshedPartitionInfo == null) { diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java index 9fd7ab66d0205..9b8c6b481f176 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java @@ -912,8 +912,7 @@ public void testHivePartitionPruneNonRefBaseTable1() throws Exception { MvTaskRunContext mvContext = processor.getMvContext(); ExecPlan execPlan = mvContext.getExecPlan(); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - Assert.assertTrue(execPlan != null); + Assert.assertNull(execPlan); } // run 5 @@ -929,15 +928,7 @@ public void testHivePartitionPruneNonRefBaseTable1() throws Exception { MvTaskRunContext mvContext = processor.getMvContext(); ExecPlan execPlan = mvContext.getExecPlan(); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, "TABLE: part_tbl1\n" + - " PARTITION PREDICATES: 4: par_date IS NOT NULL, 4: par_date >= '2020-01-01', 4: par_date < " + - "'2020-01-05'\n" + - " partitions=4/4"); - PlanTestBase.assertContains(plan, "TABLE: part_tbl2\n" + - " PARTITION PREDICATES: 8: par_date IS NOT NULL, 8: par_date >= '2020-01-01', 8: par_date < " + - "'2020-01-05'\n" + - " partitions=4/4"); + Assert.assertNull(execPlan); } starRocksAssert.dropMaterializedView("hive_partition_prune_non_ref_tables2"); @@ -1338,10 +1329,7 @@ public void testCreateMVWithMultiPartitionColumns1() throws Exception { taskRun.getProcessor(); MvTaskRunContext mvContext = processor.getMvContext(); ExecPlan execPlan = mvContext.getExecPlan(); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - // TODO(fixme): drop partition should not refresh all partitions. - PlanTestBase.assertContains(plan, " TABLE: t1_par\n"); - PlanTestBase.assertContains(plan, " partitions=6/6"); + Assert.assertTrue(execPlan == null); } starRocksAssert.dropMaterializedView("test_mv1"); } @@ -1404,9 +1392,7 @@ public void testCreateMVWithMultiPartitionColumns2() throws Exception { PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); MvTaskRunContext mvContext = processor.getMvContext(); ExecPlan execPlan = mvContext.getExecPlan(); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t1_par\n"); - PlanTestBase.assertContains(plan, " partitions=6/6"); + Assert.assertTrue(execPlan == null); } starRocksAssert.dropMaterializedView("test_mv1"); }