Merge branch 'master' into patch-2

.asf.yaml

@@ -70,16 +70,29 @@ github:
         dismiss_stale_reviews: true
         require_code_owner_reviews: true
         required_approving_review_count: 1
-    branch-1.1-lts:
+
+    branch-3.0:
       required_status_checks:
         # if strict is true, means "Require branches to be up to date before merging".
         strict: false
         contexts:
           - License Check
-
-      required_pull_request_reviews:
-        dismiss_stale_reviews: true
-        required_approving_review_count: 1
+          - Clang Formatter
+          - CheckStyle
+          - Build Broker
+          - ShellCheck
+          - Build Third Party Libraries (Linux)
+          - Build Third Party Libraries (macOS)
+          - FE UT (Doris FE UT)
+          - BE UT (Doris BE UT)
+          - Cloud UT (Doris Cloud UT)
+          - COMPILE (DORIS_COMPILE)
+          - P0 Regression (Doris Regression)
+          - External Regression (Doris External Regression)
+          - cloud_p0 (Doris Cloud Regression)
+      #required_pull_request_reviews:
+      #  dismiss_stale_reviews: true
+      #  required_approving_review_count: 1
 
     branch-2.1:
       required_status_checks:
@@ -124,6 +137,17 @@ github:
           dismiss_stale_reviews: true
           required_approving_review_count: 1
 
+    branch-1.1-lts:
+      required_status_checks:
+        # if strict is true, means "Require branches to be up to date before merging".
+        strict: false
+        contexts:
+          - License Check
+
+      required_pull_request_reviews:
+        dismiss_stale_reviews: true
+        required_approving_review_count: 1
+
   collaborators:
     - LemonLiTree
     - Yukang-Lian
@@ -134,7 +158,7 @@ github:
     - wm1581066
     - KassieZ
     - yujun777
-    - gavinchou
+    - doris-robot
 
 notifications:
     pullrequests_status:  [email protected]

.github/CODEOWNERS

@@ -18,3 +18,4 @@ be/src/io/* @platoneko @gavinchou @dataroaring
 be/src/agent/be_exec_version_manager.cpp @BiteTheDDDDt
 fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @dataroaring @CalvinKirs @morningman
 **/pom.xml @CalvinKirs @morningman
+fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @dataroaring @morningman @yiguolei @xiaokang

.github/workflows/auto-cherry-pick.yml

@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Auto Cherry-Pick to Branch
+
+on:
+  pull_request_target:
+    types:
+      - closed
+    branches:
+      - master
+permissions:
+  checks: write
+  contents: write
+  pull-requests: write
+jobs:
+  auto_cherry_pick:
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') && github.event.pull_request.merged == true }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          pip install PyGithub
+      - name: Check SHA
+        run: |
+            expected_sha="80b7c6087f2a3e4f4c7f035a52e8e7b05ce00f27aa5c1bd52179df685c912447f94a96145fd3204a3958d8ed9777de5a5183b120e99e0e95bbca0366d69b0ac0"
+            calculated_sha=$(sha512sum tools/auto-pick-script.py | awk '{ print $1 }')
+            if [ "$calculated_sha" != "$expected_sha" ]; then
+              echo "SHA mismatch! Expected: $expected_sha, but got: $calculated_sha"
+              exit 1
+            else
+              echo "SHA matches: $calculated_sha"
+            fi
+      - name: Auto cherry-pick
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_NAME: ${{ github.repository }}
+          CONFLICT_LABEL: cherry-pick-conflict-in-3.0
+        run: |
+          python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-3.0  

.github/workflows/code-checks.yml

@@ -105,7 +105,7 @@ jobs:
             popd
 
             export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
-            DISABLE_BE_JAVA_EXTENSIONS=ON DO_NOT_CHECK_JAVA_ENV=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be --cloud
+            DISABLE_BE_JAVA_EXTENSIONS=ON DO_NOT_CHECK_JAVA_ENV=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be
           fi
 
           echo "should_check=${{ steps.filter.outputs.cpp_changes }}" >>${GITHUB_OUTPUT}

.github/workflows/scope-label.yml

@@ -18,11 +18,12 @@
 ---
 name: Add Scope Labeler
 
-on:
-  pull_request_target:
-    types:
-      - opened
-      - synchronize
+# This action has some error, skip it temporarily
+#on:
+#  pull_request_target:
+#    types:
+#      - opened
+#      - synchronize
 
 jobs:
   process:

.github/workflows/sonarcloud.yml

@@ -41,11 +41,11 @@ jobs:
               - 'fe/**'
               - 'gensrc/proto/**'
               - 'gensrc/thrift/**'
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         if: ${{ steps.filter.outputs.fe_changes == 'true' }}
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
           distribution: 'adopt'
       - name: Cache SonarCloud packages
         if: ${{ steps.filter.outputs.fe_changes == 'true' }}

.gitignore

@@ -56,6 +56,7 @@ thirdparty/doris-thirdparty*.tar.xz
 docker/thirdparties/docker-compose/mysql/data
 docker/thirdparties/docker-compose/hive/scripts/tpch1.db/
 docker/thirdparties/docker-compose/hive/scripts/paimon1
+docker/thirdparties/docker-compose/hive/scripts/tvf_data
 
 fe_plugins/output
 fe_plugins/**/.factorypath

be/CMakeLists.txt

@@ -299,8 +299,6 @@ if (COMPILER_CLANG)
                         -Wno-implicit-float-conversion
                         -Wno-implicit-int-conversion
                         -Wno-sign-conversion
-                        -Wno-missing-field-initializers
-                        -Wno-unused-const-variable
                         -Wno-shorten-64-to-32)
     if (USE_LIBCPP)
         add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
@@ -784,6 +782,7 @@ install(DIRECTORY DESTINATION ${OUTPUT_DIR}/conf)
 install(FILES
     ${BASE_DIR}/../bin/start_be.sh
     ${BASE_DIR}/../bin/stop_be.sh
+    ${BASE_DIR}/../tools/jeprof
     PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
     GROUP_READ GROUP_WRITE GROUP_EXECUTE
     WORLD_READ WORLD_EXECUTE

be/src/agent/be_exec_version_manager.cpp

@@ -17,12 +17,9 @@
 
 #include "agent/be_exec_version_manager.h"
 
-namespace doris {
+#include "common/exception.h"
 
-const std::map<int, const std::set<std::string>> AGGREGATION_CHANGE_MAP = {
-        {AGGREGATION_2_1_VERSION,
-         {"window_funnel", "stddev_samp", "variance_samp", "percentile_approx_weighted",
-          "percentile_approx", "covar_samp", "percentile", "percentile_array"}}};
+namespace doris {
 
 Status BeExecVersionManager::check_be_exec_version(int be_exec_version) {
     if (be_exec_version > max_be_exec_version || be_exec_version < min_be_exec_version) {
@@ -35,19 +32,42 @@ Status BeExecVersionManager::check_be_exec_version(int be_exec_version) {
     return Status::OK();
 }
 
-void BeExecVersionManager::check_agg_state_compatibility(int current_be_exec_version,
-                                                         int data_be_exec_version,
-                                                         std::string function_name) {
-    if (current_be_exec_version > AGGREGATION_2_1_VERSION &&
-        data_be_exec_version <= AGGREGATION_2_1_VERSION &&
-        AGGREGATION_CHANGE_MAP.find(AGGREGATION_2_1_VERSION)->second.contains(function_name)) {
+int BeExecVersionManager::get_function_compatibility(int be_exec_version,
+                                                     std::string function_name) {
+    if (_function_restrict_map.contains(function_name) && be_exec_version != get_newest_version()) {
         throw Exception(Status::InternalError(
-                "agg state data with {} is not supported, "
-                "current_be_exec_version={}, data_be_exec_version={}, need to rebuild the data "
-                "or set the be_exec_version={} in fe.conf",
-                function_name, current_be_exec_version, data_be_exec_version,
-                AGGREGATION_2_1_VERSION));
+                "function {} do not support old be exec version, maybe it's because doris are "
+                "doing a rolling upgrade. newest_version={}, input_be_exec_version={}",
+                function_name, get_newest_version(), be_exec_version));
+    }
+
+    auto it = _function_change_map.find(function_name);
+    if (it == _function_change_map.end()) {
+        // 0 means no compatibility issues need to be dealt with
+        return 0;
     }
+
+    auto version_it = it->second.lower_bound(be_exec_version);
+    if (version_it == it->second.end()) {
+        return 0;
+    }
+
+    return *version_it;
+}
+
+void BeExecVersionManager::check_function_compatibility(int current_be_exec_version,
+                                                        int data_be_exec_version,
+                                                        std::string function_name) {
+    if (get_function_compatibility(current_be_exec_version, function_name) ==
+        get_function_compatibility(data_be_exec_version, function_name)) {
+        return;
+    }
+
+    throw Exception(Status::InternalError(
+            "agg state data with {} is not supported, "
+            "current_be_exec_version={}, data_be_exec_version={}, need to rebuild the data "
+            "or set the be_exec_version={} in fe.conf temporary",
+            function_name, current_be_exec_version, data_be_exec_version, data_be_exec_version));
 }
 
 /**
@@ -69,7 +89,7 @@ void BeExecVersionManager::check_agg_state_compatibility(int current_be_exec_ver
  * 3: start from doris 2.0.0 (by some mistakes)
  *    a. aggregation function do not serialize bitmap to string.
  *    b. support window funnel mode.
- * 4/5: start from doris 2.1.0
+ * 4: start from doris 2.1.0
  *    a. ignore this line, window funnel mode should be enabled from 2.0.
  *    b. array contains/position/countequal function return nullable in less situations.
  *    c. cleared old version of Version 2.
@@ -79,14 +99,22 @@ void BeExecVersionManager::check_agg_state_compatibility(int current_be_exec_ver
  *    g. do local merge of remote runtime filter
  *    h. "now": ALWAYS_NOT_NULLABLE -> DEPEND_ON_ARGUMENTS
  *
- * 7: start from doris 3.0.0
+ * 5: start from doris 3.0.0
+ *    a. change some agg function nullable property: PR #37215
+ *
+ * 6: start from doris 3.0.1 and 2.1.6
  *    a. change the impl of percentile (need fix)
  *    b. clear old version of version 3->4
  *    c. change FunctionIsIPAddressInRange from AlwaysNotNullable to DependOnArguments
- *    d. change some agg function nullable property: PR #37215
- *    e. change variant serde to fix PR #38413
+ *    d. change variant serde to fix PR #38413
+ *
+ * 7: start from doris 3.0.2
+ *    a. window funnel logic change
+*     b. support const column in serialize/deserialize function: PR #41175
  */
-const int BeExecVersionManager::max_be_exec_version = 7;
-const int BeExecVersionManager::min_be_exec_version = 0;
 
+const int BeExecVersionManager::max_be_exec_version = 8;
+const int BeExecVersionManager::min_be_exec_version = 0;
+std::map<std::string, std::set<int>> BeExecVersionManager::_function_change_map {};
+std::set<std::string> BeExecVersionManager::_function_restrict_map;
 } // namespace doris