[Enhancement] Skip rebalancing scan ranges for hdfs backend selector …

…by default when using datacache. (#51996) Signed-off-by: GavinMar <[email protected]>
StarRocks · Oct 18, 2024 · fe00c0b · fe00c0b
1 parent f56aac4
commit fe00c0b
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 1 deletion.
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/HDFSBackendSelector.java b/fe/fe-core/src/main/java/com/starrocks/qe/HDFSBackendSelector.java
@@ -171,6 +171,15 @@ private ComputeNode reBalanceScanRangeForComputeNode(List<ComputeNode> backends,
  return null;
  }
 
+ boolean forceReBalance = ConnectContext.get() != null ? ConnectContext.get().getSessionVariable().
+ getHdfsBackendSelectorForceRebalance() : false;
+ boolean enableDataCache = ConnectContext.get() != null ? ConnectContext.get().getSessionVariable().
+ isEnableScanDataCache() : false;
+ // If force-rebalancing is not specified and cache is used, skip the rebalancing directly.
+ if (!forceReBalance && enableDataCache) {
+ return backends.get(0);
+ }
+
  ComputeNode node = null;
  long addedScans = scanRangeLocations.scan_range.hdfs_scan_range.length;
  for (ComputeNode backend : backends) {
@@ -349,4 +358,4 @@ private void recordScanRangeStatistic() {
  Tracers.count(Tracers.Module.EXTERNAL, key, value);
  }
  }
-}
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
@@ -717,6 +717,8 @@ public static MaterializedViewRewriteMode parse(String str) {
 
  public static final String HDFS_BACKEND_SELECTOR_HASH_ALGORITHM = "hdfs_backend_selector_hash_algorithm";
 
+ public static final String HDFS_BACKEND_SELECTOR_FORCE_REBALANCE = "hdfs_backend_selector_force_rebalance";
+
  public static final String CONSISTENT_HASH_VIRTUAL_NUMBER = "consistent_hash_virtual_number";
 
  public static final String ENABLE_COLLECT_TABLE_LEVEL_SCAN_STATS = "enable_collect_table_level_scan_stats";
@@ -1545,6 +1547,9 @@ public static MaterializedViewRewriteMode parse(String str) {
  @VariableMgr.VarAttr(name = HDFS_BACKEND_SELECTOR_HASH_ALGORITHM, flag = VariableMgr.INVISIBLE)
  private String hdfsBackendSelectorHashAlgorithm = "consistent";
 
+ @VariableMgr.VarAttr(name = HDFS_BACKEND_SELECTOR_FORCE_REBALANCE, flag = VariableMgr.INVISIBLE)
+ private boolean hdfsBackendSelectorForceRebalance = false;
+
  @VariableMgr.VarAttr(name = CONSISTENT_HASH_VIRTUAL_NUMBER, flag = VariableMgr.INVISIBLE)
  private int consistentHashVirtualNodeNum = 256;
 
@@ -2723,6 +2728,14 @@ public void setHdfsBackendSelectorHashAlgorithm(String hdfsBackendSelectorHashAl
  this.hdfsBackendSelectorHashAlgorithm = hdfsBackendSelectorHashAlgorithm;
  }
 
+ public boolean getHdfsBackendSelectorForceRebalance() {
+ return hdfsBackendSelectorForceRebalance;
+ }
+
+ public void setHdfsBackendSelectorForceRebalance(boolean hdfsBackendSelectorForceRebalance) {
+ this.hdfsBackendSelectorForceRebalance = hdfsBackendSelectorForceRebalance;
+ }
+
  public int getConsistentHashVirtualNodeNum() {
  return consistentHashVirtualNodeNum;
  }