Merge branch 'unstable' into batch_query

OpenAtomFoundation · Jul 10, 2024 · e2ed7ba · e2ed7ba
2 parents 1b5bfbe + 1de6598
commit e2ed7ba
Show file tree

Hide file tree

Showing 10 changed files with 21 additions and 196 deletions.
diff --git a/.github/workflows/pika.yml b/.github/workflows/pika.yml
@@ -78,146 +78,6 @@ jobs:
           chmod +x integrate_test.sh
           sh integrate_test.sh
 
-  build_on_centos:
-    runs-on: ubuntu-latest
-    container:
-      image: centos:7
-
-    steps:
-      - name: set up mirror
-        run: |
-          rm -rf /etc/yum.repos.d/CentOS-Base.repo 
-          cat > /etc/yum.repos.d/CentOS-Base.repo << EOL
-          [base]
-          name=CentOS-\$releasever - Base
-          baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/os/\$basearch/
-          gpgcheck=1
-          gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-          
-          [updates]
-          name=CentOS-\$releasever - Updates
-          baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/updates/\$basearch/
-          gpgcheck=1
-          gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-          
-          [extras]
-          name=CentOS-\$releasever - Extras
-          baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/extras/\$basearch/
-          gpgcheck=1
-          gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-          
-          [centosplus]
-          name=CentOS-\$releasever - Plus
-          baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/centosplus/\$basearch/
-          gpgcheck=1
-          enabled=0
-          gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-          EOL
-          
-          cat > /etc/yum.repos.d/CentOS-SCLo-scl.repo << EOL
-          [centos-sclo-sclo]
-          name=CentOS-7 - SCLo sclo
-          baseurl=https://mirrors.aliyun.com/centos/7/sclo/x86_64/sclo/
-          gpgcheck=1
-          enabled=1
-          gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo
-          
-          [centos-sclo-sclo-source]
-          name=CentOS-7 - SCLo sclo Source
-          baseurl=https://mirrors.aliyun.com/centos/7/sclo/Source/sclo/
-          gpgcheck=1
-          enabled=0
-          gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo
-          EOL
-          
-          cat > /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo << EOL
-          [centos-sclo-rh]
-          name=CentOS-7 - SCLo rh
-          baseurl=https://mirrors.aliyun.com/centos/7/sclo/x86_64/rh/
-          gpgcheck=1
-          enabled=1
-          gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo
-          
-          [centos-sclo-rh-source]
-          name=CentOS-7 - SCLo rh Source
-          baseurl=https://mirrors.aliyun.com/centos/7/sclo/Source/rh/
-          gpgcheck=1
-          enabled=0
-          gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo
-          EOL
-          
-          rpm --import https://www.centos.org/keys/RPM-GPG-KEY-CentOS-7
-          rpm --import https://www.centos.org/keys/RPM-GPG-KEY-CentOS-SIG-SCLo
-          
-          yum clean all
-          yum makecache
-
-      - name: Install deps
-        run: |
-          yum install -y wget git autoconf centos-release-scl gcc
-          yum install -y devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-make devtoolset-10-bin-util
-          yum install -y llvm-toolset-7 llvm-toolset-7-clang tcl which 
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-
-      - name: Install cmake
-        run: |
-          wget https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-x86_64.sh
-          bash ./cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
-
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Configure CMake
-        run: |
-          source /opt/rh/devtoolset-10/enable
-          cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
-
-      - uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/deps
-          key: ${{ runner.os }}-centos-deps-${{ hashFiles('**/CMakeLists.txt') }}
-
-      - uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/buildtrees
-          key: ${{ runner.os }}-centos-buildtrees-${{ hashFiles('**/CMakeLists.txt') }}
-
-      - name: Build
-        run: |
-          source /opt/rh/devtoolset-10/enable
-          cmake --build build --config ${{ env.BUILD_TYPE }}
-
-      - name: Test
-        working-directory: ${{ github.workspace }}/build
-        run: ctest -C ${{ env.BUILD_TYPE }}
-
-      - name: Unit Test
-        working-directory: ${{ github.workspace }}
-        run: ./pikatests.sh all
-
-      - name: Start codis, pika master and pika slave
-        working-directory: ${{ github.workspace }}/build
-        run: |
-          chmod +x ../tests/integration/start_master_and_slave.sh
-          ../tests/integration/start_master_and_slave.sh
-          chmod +x ../tests/integration/start_codis.sh
-          ../tests/integration/start_codis.sh
-
-      - name: Run Go E2E Tests
-        working-directory: ${{ github.workspace }}/build
-        run: |
-          cd ../tools/pika_keys_analysis/
-          go test -v ./...
-          cd ../../tests/integration/
-          chmod +x integrate_test.sh
-          sh integrate_test.sh
-
   build_on_macos:
     runs-on: macos-12
 

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -47,53 +47,9 @@ jobs:
           name: ${{ matrix.name }}.sha256sum
           path: build/${{ matrix.name }}.sha256sum
 
-  centos:
-    runs-on: ubuntu-latest
-    container:
-      image: centos:7
-    env:
-      name: ${{ github.event.repository.name }}-${{ github.ref_name }}-centos-amd64.tar.gz
-    steps:
-      - name: Install deps
-        run: |
-          yum install -y wget git autoconf centos-release-scl gcc perl-Digest-SHA
-          yum install -y devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-make devtoolset-10-bin-util
-          yum install -y llvm-toolset-7 llvm-toolset-7-clang tcl which 
-
-      - name: Install cmake
-        run: |
-          wget https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-x86_64.sh
-          bash ./cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
-
-      - name: Checkout sources
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Release build os - centos
-        run: |
-          chmod +x ci/release-build.sh
-          ./ci/release-build.sh install centos ${{ env.BUILD_TYPE }} -xe
-
-      - name: Calculate checksum and rename binary
-        shell: bash
-        run: ./ci/release-build.sh checksum ${{ github.event.repository.name }} ${{ env.name }}
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.name }}
-          path: build/${{ env.name }}
-
-      - name: Upload checksum of artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.name }}.sha256sum
-          path: build/${{ env.name }}.sha256sum
-
   release:
     name: Release artifacts
-    needs: [ build,centos ]
+    needs: [ build ]
     runs-on: ubuntu-latest
     steps:
       - name: Download artifacts

diff --git a/CHANGELOG.MD b/CHANGELOG.MD
@@ -44,6 +44,8 @@
 
 ## Bugfix
 
+- Solve the issue where failing to destruct 'iter' causes 'pkpatternmatchdel' not to delete 'iter' before returning, potentially leading to RocksDB perpetually referencing a version, causing data inconsistency.[#2785](https://github.com/OpenAtomFoundation/pika/pull/2785)@[wangshao1](https://github.com/wangshao1)
+
 - Fixed an issue with parsing the config parameter min-blob-size when it includes units.[#2767](https://github.com/OpenAtomFoundation/pika/pull/2767)@[wangshao1](https://github.com/wangshao1)
 
 - Fixed an issue with abnormal return values in ZREVRANK.[#2763](https://github.com/OpenAtomFoundation/pika/pull/2763)@[chejinge](https://github.com/chejinge)

diff --git a/CHANGELOG_CN.MD b/CHANGELOG_CN.MD
@@ -44,6 +44,8 @@
 
 ## Bugfix
 
+- 修复 iter 未被析构，导致 pkpatternmatchdel 在返回之前不会删除 iter，这可能会导致 rocksdb 永远引用一个版本，导致数据不符合预期的问题[#2785](https://github.com/OpenAtomFoundation/pika/pull/2785)@[wangshao1](https://github.com/wangshao1)
+
 - 修复 config 参数 min-blob-size 带单位时解析错误的问题[#2767](https://github.com/OpenAtomFoundation/pika/pull/2767)@[wangshao1](https://github.com/wangshao1)
 
 - 修复 zverank 返回值异常的问题[#2763](https://github.com/OpenAtomFoundation/pika/pull/2763)@[chejinge](https://github.com/chejinge)
@@ -88,7 +90,7 @@
 
 - 发版支持自动打包 centos7 和 centos8 平台的二进制编译包[#2535](https://github.com/OpenAtomFoundation/pika/pull/2535)@[baerwang](https://github.com/baerwang)
 
-- 修复Codis 侧的 getrange 命令没有返回预期结果的问题[#2510](https://github.com/OpenAtomFoundation/pika/pull/2510)@[luky116](https://github.com/luky116)
+- 修复 Codis 侧的 getrange 命令没有返回预期结果的问题[#2510](https://github.com/OpenAtomFoundation/pika/pull/2510)@[luky116](https://github.com/luky116)
 
 
 # v3.5.4

diff --git a/ci/release-build.sh b/ci/release-build.sh
@@ -22,9 +22,6 @@ function configure_cmake() {
     cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE
   elif [[ $OS == *"ubuntu"* ]]; then
     cmake -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS="-s" -DCMAKE_EXE_LINKER_FLAGS="-s"
-  elif [[ $OS == *"centos"* ]]; then
-    source /opt/rh/devtoolset-10/enable
-    cmake -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
   fi
   echo "configure cmake after ..."
 }

diff --git a/codis/pkg/topom/topom_group.go b/codis/pkg/topom/topom_group.go
@@ -391,8 +391,8 @@ func (s *Topom) tryFixReplicationRelationship(group *models.Group, groupServer *
 			return nil
 		}
 
-		// current server is slave, execute the command `slaveof [new master ip] [new master port]`
-		if err = updateMasterToNewOne(groupServer.Addr, curMasterAddr, s.config.ProductAuth); err != nil {
+		// current server is slave, execute the command `slaveof [new master ip] [new master port] force`
+		if err = updateMasterToNewOneForcefully(groupServer.Addr, curMasterAddr, s.config.ProductAuth); err != nil {
 			return err
 		}
 	}

diff --git a/src/net/src/dispatch_thread.cc b/src/net/src/dispatch_thread.cc
@@ -64,10 +64,10 @@ int DispatchThread::StartThread() {
   }
 
   // Adding timer tasks and run timertaskThread
-  timerTaskThread_.AddTimerTask("blrpop_blocking_info_scan", 250, true,
+  timer_task_thread_.AddTimerTask("blrpop_blocking_info_scan", 250, true,
                                 [this] { this->ScanExpiredBlockedConnsOfBlrpop(); });
-  timerTaskThread_.set_thread_name("TimerTaskThread");
-  timerTaskThread_.StartThread();
+  timer_task_thread_.set_thread_name("TimerTaskThread");
+  timer_task_thread_.StartThread();
   return ServerThread::StartThread();
 }
 
@@ -88,7 +88,7 @@ int DispatchThread::StopThread() {
       worker_thread_[i]->private_data_ = nullptr;
     }
   }
-  timerTaskThread_.StopThread();
+  timer_task_thread_.StopThread();
   return ServerThread::StopThread();
 }
 

diff --git a/src/net/src/dispatch_thread.h b/src/net/src/dispatch_thread.h
@@ -161,7 +161,7 @@ class DispatchThread : public ServerThread {
    */
   std::shared_mutex block_mtx_;
 
-  TimerTaskThread timerTaskThread_;
+  TimerTaskThread timer_task_thread_;
 };  // class DispatchThread
 
 }  // namespace net

diff --git a/src/net/src/net_util.h b/src/net/src/net_util.h
@@ -47,6 +47,12 @@ struct ExecTsWithId {
   }
 };
 
+/*
+ * For simplicity, current version of TimerTaskThread has no lock inside and all task should be registered before TimerTaskThread started,
+ * but if you have the needs of dynamically add/remove timer task after TimerTaskThread started, you can simply add a mutex to protect
+ * the timer_task_manager_ and also a pipe to wake up the maybe being endless-wait epoll(if all task consumed, epoll will sink into
+ * endless wait) to implement the feature.
+ */
 class TimerTaskManager {
  public:
   TimerTaskManager() = default;

diff --git a/src/storage/src/redis_strings.cc b/src/storage/src/redis_strings.cc
@@ -1748,6 +1748,7 @@ rocksdb::Status Redis::PKPatternMatchDel(const std::string& pattern, int32_t* re
         batch.Clear();
       } else {
         *ret = total_delete;
+        delete iter;
         return s;
       }
     }
@@ -1761,6 +1762,7 @@ rocksdb::Status Redis::PKPatternMatchDel(const std::string& pattern, int32_t* re
     }
   }
 
+  delete iter;
   *ret = total_delete;
   return s;
 }