From d02c72f7e508a0ea68ff84f8aa37ded03986d6a3 Mon Sep 17 00:00:00 2001 From: AndyZiYe Date: Wed, 25 Dec 2024 22:03:21 +0800 Subject: [PATCH] Revert "53787 53881" --- .../starrocks/catalog/ListPartitionInfo.java | 12 ------- .../java/com/starrocks/qe/Coordinator.java | 4 ++- .../rewrite/OptOlapPartitionPruner.java | 2 +- .../transformation/ListPartitionPruner.java | 36 ++----------------- 4 files changed, 6 insertions(+), 48 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java b/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java index cae19b22c132a..0c83d181e3875 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java @@ -213,18 +213,6 @@ public Map> getIdToValues() { return idToValues; } - public boolean isSingleValuePartition(long id) { - List values = getIdToValues().get(id); - if (values != null && values.size() == 1) { - return true; - } - List> multiValues = getIdToMultiValues().get(id); - if (multiValues != null && multiValues.size() == 1) { - return true; - } - return false; - } - /** * serialize data to log * diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/Coordinator.java b/fe/fe-core/src/main/java/com/starrocks/qe/Coordinator.java index 1bebe44964bca..26991d3315f60 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/Coordinator.java @@ -196,7 +196,7 @@ public class Coordinator { private final boolean isBlockQuery; private int numReceivedRows = 0; private List deltaUrls; - private final Map loadCounters = new HashMap<>(); + private Map loadCounters; private String trackingUrl; private final Set rejectedRecordPaths = new HashSet<>(); // for export @@ -246,6 +246,7 @@ public Coordinator(StreamLoadPlanner planner, TNetworkAddress address) { attachInstanceProfileToFragmentProfile(); deltaUrls = Lists.newArrayList(); + loadCounters = Maps.newHashMap(); this.connectContext = planner.getConnectContext(); // for complie @@ -714,6 +715,7 @@ private void prepareResultSink() throws Exception { queryOptions.setRuntime_profile_report_interval(30); } deltaUrls = Lists.newArrayList(); + loadCounters = Maps.newHashMap(); List relatedBackendIds = Lists.newArrayList(coordinatorPreprocessor.getAddressToBackendID().values()); GlobalStateMgr.getCurrentState().getLoadMgr() .initJobProgress(jobId, queryId, coordinatorPreprocessor.getInstanceIds(), diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptOlapPartitionPruner.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptOlapPartitionPruner.java index 3ee1f04fdf9e0..daabab56b260d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptOlapPartitionPruner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/OptOlapPartitionPruner.java @@ -307,7 +307,7 @@ private static List listPartitionPrune(OlapTable olapTable, ListPartitionI List scalarOperatorList = Utils.extractConjuncts(operator.getPredicate()); PartitionPruner partitionPruner = new ListPartitionPruner(columnToPartitionValuesMap, - columnToNullPartitions, scalarOperatorList, specifyPartitionIds, listPartitionInfo); + columnToNullPartitions, scalarOperatorList, specifyPartitionIds); try { List prune = partitionPruner.prune(); if (prune == null && isTemporaryPartitionPrune) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java index 39ad813d870e9..25a9fbd4f9999 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java @@ -20,7 +20,6 @@ import com.starrocks.analysis.BinaryType; import com.starrocks.analysis.Expr; import com.starrocks.analysis.LiteralExpr; -import com.starrocks.catalog.ListPartitionInfo; import com.starrocks.catalog.Type; import com.starrocks.common.AnalysisException; import com.starrocks.common.Pair; @@ -84,27 +83,17 @@ public class ListPartitionPruner implements PartitionPruner { private final Set allPartitions; private final List partitionColumnRefs; private final List specifyPartitionIds; - private final ListPartitionInfo listPartitionInfo; public ListPartitionPruner( Map>> columnToPartitionValuesMap, Map> columnToNullPartitions, List partitionConjuncts, List specifyPartitionIds) { - this(columnToPartitionValuesMap, columnToNullPartitions, partitionConjuncts, specifyPartitionIds, null); - } - - public ListPartitionPruner( - Map>> columnToPartitionValuesMap, - Map> columnToNullPartitions, - List partitionConjuncts, List specifyPartitionIds, - ListPartitionInfo listPartitionInfo) { this.columnToPartitionValuesMap = columnToPartitionValuesMap; this.columnToNullPartitions = columnToNullPartitions; this.partitionConjuncts = partitionConjuncts; this.allPartitions = getAllPartitions(); this.partitionColumnRefs = getPartitionColumnRefs(); this.specifyPartitionIds = specifyPartitionIds; - this.listPartitionInfo = listPartitionInfo; } private Set getAllPartitions() { @@ -346,17 +335,7 @@ private Set evalBinaryPredicate(BinaryPredicateOperator binaryPredicate) { matches.removeAll(nullPartitions); // remove partition matches literal if (partitionValueMap.containsKey(literal)) { - if (listPartitionInfo == null) { - // external table - matches.removeAll(partitionValueMap.get(literal)); - } else { - Set partitionIds = partitionValueMap.get(literal); - for (Long id : partitionIds) { - if (listPartitionInfo.isSingleValuePartition(id)) { - matches.remove(id); - } - } - } + matches.removeAll(partitionValueMap.get(literal)); } return matches; case LE: @@ -469,18 +448,7 @@ private Set evalInPredicate(InPredicateOperator inPredicate) { Set partitions = partitionValueMap.get(literal); if (partitions != null) { if (inPredicate.isNotIn()) { - // external table, one partition column for one partition can only have one value - if (listPartitionInfo == null) { - matches.removeAll(partitions); - } else { - // for olap table, if one partition is multi value partition like PARTITION pCalifornia VALUES IN ("Los Angeles","San Francisco","San Diego") - // and we have a not in predicate like city not in ("Los Angeles"), it's not safe to remove this partition - for (Long id : partitions) { - if (listPartitionInfo.isSingleValuePartition(id)) { - matches.remove(id); - } - } - } + matches.removeAll(partitions); } else { matches.addAll(partitions); }