[Enhancement] Skip rebalancing scan ranges for hdfs backend selector …

…by default when using datacache. (backport #51996) (#52072) Co-authored-by: Gavin <[email protected]>
StarRocks · Oct 18, 2024 · 0666b60 · 0666b60
1 parent 8c85bc3
commit 0666b60
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 1 deletion.
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/HDFSBackendSelector.java b/fe/fe-core/src/main/java/com/starrocks/qe/HDFSBackendSelector.java
@@ -169,6 +169,15 @@ private ComputeNode reBalanceScanRangeForComputeNode(List<ComputeNode> backends,
  return null;
  }
 
+ boolean forceReBalance = ConnectContext.get() != null ? ConnectContext.get().getSessionVariable().
+ getHdfsBackendSelectorForceRebalance() : false;
+ boolean enableDataCache = ConnectContext.get() != null ? ConnectContext.get().getSessionVariable().
+ isEnableScanDataCache() : false;
+ // If force-rebalancing is not specified and cache is used, skip the rebalancing directly.
+ if (!forceReBalance && enableDataCache) {
+ return backends.get(0);
+ }
+
  ComputeNode node = null;
  long addedScans = scanRangeLocations.scan_range.hdfs_scan_range.length;
  for (ComputeNode backend : backends) {
@@ -323,4 +332,4 @@ private void recordScanRangeStatistic() {
  }
  Tracers.record(Tracers.Module.EXTERNAL, scanNode.getTableName() + " rebalance_bytes", sb.toString());
  }
-}
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
@@ -697,6 +697,8 @@ public static MaterializedViewRewriteMode parse(String str) {
 
  public static final String HDFS_BACKEND_SELECTOR_HASH_ALGORITHM = "hdfs_backend_selector_hash_algorithm";
 
+ public static final String HDFS_BACKEND_SELECTOR_FORCE_REBALANCE = "hdfs_backend_selector_force_rebalance";
+
  public static final String CONSISTENT_HASH_VIRTUAL_NUMBER = "consistent_hash_virtual_number";
 
  public static final String ENABLE_COLLECT_TABLE_LEVEL_SCAN_STATS = "enable_collect_table_level_scan_stats";
@@ -1494,6 +1496,9 @@ public static MaterializedViewRewriteMode parse(String str) {
  @VariableMgr.VarAttr(name = HDFS_BACKEND_SELECTOR_HASH_ALGORITHM, flag = VariableMgr.INVISIBLE)
  private String hdfsBackendSelectorHashAlgorithm = "consistent";
 
+ @VariableMgr.VarAttr(name = HDFS_BACKEND_SELECTOR_FORCE_REBALANCE, flag = VariableMgr.INVISIBLE)
+ private boolean hdfsBackendSelectorForceRebalance = false;
+
  @VariableMgr.VarAttr(name = CONSISTENT_HASH_VIRTUAL_NUMBER, flag = VariableMgr.INVISIBLE)
  private int consistentHashVirtualNodeNum = 256;
 
@@ -2592,6 +2597,14 @@ public void setHdfsBackendSelectorHashAlgorithm(String hdfsBackendSelectorHashAl
  this.hdfsBackendSelectorHashAlgorithm = hdfsBackendSelectorHashAlgorithm;
  }
 
+ public boolean getHdfsBackendSelectorForceRebalance() {
+ return hdfsBackendSelectorForceRebalance;
+ }
+
+ public void setHdfsBackendSelectorForceRebalance(boolean hdfsBackendSelectorForceRebalance) {
+ this.hdfsBackendSelectorForceRebalance = hdfsBackendSelectorForceRebalance;
+ }
+
  public int getConsistentHashVirtualNodeNum() {
  return consistentHashVirtualNodeNum;
  }