Merge branch 'OpenAtomFoundation:unstable' into 240624_use_fwrite_binlog

.github/workflows/pika.yml

@@ -78,6 +78,72 @@ jobs:
           chmod +x integrate_test.sh
           sh integrate_test.sh
 
+  build_on_rocky:
+    runs-on: ubuntu-latest
+    container:
+      image: rockylinux:9
+
+    steps:
+      - name: Install deps
+        run: |
+          dnf update -y
+          dnf install -y bash cmake wget git autoconf gcc perl-Digest-SHA tcl which tar g++ tar epel-release gcc-c++ libstdc++-devel gcc-toolset-13
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Configure CMake
+        run: |
+          source /opt/rh/gcc-toolset-13/enable
+          cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address .
+
+      - uses: actions/cache@v3
+        with:
+          path: ${{ github.workspace }}/deps
+          key: ${{ runner.os }}-rocky-deps-${{ hashFiles('**/CMakeLists.txt') }}
+
+      - uses: actions/cache@v3
+        with:
+          path: ${{ github.workspace }}/buildtrees
+          key: ${{ runner.os }}-rocky-buildtrees-${{ hashFiles('**/CMakeLists.txt') }}
+
+      - name: Build
+        run: |
+          source /opt/rh/gcc-toolset-13/enable
+          cmake --build build --config ${{ env.BUILD_TYPE }}
+
+      - name: Test
+        working-directory: ${{ github.workspace }}/build
+        run: ctest -C ${{ env.BUILD_TYPE }}
+
+      - name: Unit Test
+        working-directory: ${{ github.workspace }}
+        run: ./pikatests.sh all
+
+      - name: Start codis, pika master and pika slave
+        working-directory: ${{ github.workspace }}/build
+        run: |
+          chmod +x ../tests/integration/start_master_and_slave.sh
+          ../tests/integration/start_master_and_slave.sh
+          chmod +x ../tests/integration/start_codis.sh
+          ../tests/integration/start_codis.sh
+
+      - name: Run Go E2E Tests
+        working-directory: ${{ github.workspace }}/build
+        run: |
+          cd ../tools/pika_keys_analysis/
+          go test -v ./...
+          cd ../../tests/integration/
+          chmod +x integrate_test.sh
+          sh integrate_test.sh
+
   build_on_macos:
     runs-on: macos-12
 

.github/workflows/release.yml

@@ -47,9 +47,47 @@ jobs:
           name: ${{ matrix.name }}.sha256sum
           path: build/${{ matrix.name }}.sha256sum
 
+  rocky:
+    runs-on: ubuntu-latest
+    container:
+      image: rockylinux:9
+    env:
+      name: ${{ github.event.repository.name }}-${{ github.ref_name }}-rocky-amd64.tar.gz
+    steps:
+      - name: Install deps
+        run: |
+          dnf update -y
+          dnf install -y bash cmake wget git autoconf gcc perl-Digest-SHA tcl which tar g++ tar epel-release gcc-c++ libstdc++-devel gcc-toolset-13
+
+      - name: Checkout sources
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Release build os - rocky
+        run: |
+          chmod +x ci/release-build.sh
+          ./ci/release-build.sh install rocky ${{ env.BUILD_TYPE }} -xe
+
+      - name: Calculate checksum and rename binary
+        shell: bash
+        run: ./ci/release-build.sh checksum ${{ github.event.repository.name }} ${{ env.name }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.name }}
+          path: build/${{ env.name }}
+
+      - name: Upload checksum of artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.name }}.sha256sum
+          path: build/${{ env.name }}.sha256sum
+
   release:
     name: Release artifacts
-    needs: [ build ]
+    needs: [ build, rocky ]
     runs-on: ubuntu-latest
     steps:
       - name: Download artifacts

CMakeLists.txt

@@ -22,6 +22,7 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   endif()
 endif()
 
+link_directories("/opt/rh/gcc-toolset-13/root/lib/gcc/x86_64-redhat-linux/13")
 
 ############# You should enable sanitizer if you are developing pika #############
 # Uncomment the following two lines to enable AddressSanitizer to detect memory leaks and other memory-related bugs.
@@ -159,7 +160,7 @@ ExternalProject_Add(gtest
   make -j${CPU_CORE}
 )
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
   set(GTEST_LIBRARY ${INSTALL_LIBDIR_64}/libgtest.a)
   set(GTEST_MAIN_LIBRARY ${INSTALL_LIBDIR_64}/libgtest_main.a)
   set(GMOCK_LIBRARY ${INSTALL_LIBDIR_64}/libgmock.a)
@@ -282,7 +283,7 @@ else()
   set(LIB_GLOG libglog.a)
 endif()
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
   set(GLOG_LIBRARY ${INSTALL_LIBDIR_64}/${LIB_GLOG})
 else()
   set(GLOG_LIBRARY ${INSTALL_LIBDIR}/${LIB_GLOG})
@@ -317,7 +318,7 @@ ExternalProject_Add(snappy
   make -j${CPU_CORE}
 )
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
   set(SNAPPY_LIBRARY ${INSTALL_LIBDIR_64}/libsnappy.a)
 else()
   set(SNAPPY_LIBRARY ${INSTALL_LIBDIR}/libsnappy.a)
@@ -355,7 +356,7 @@ ExternalProject_Add(zstd
   make -j${CPU_CORE}
 )
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
   set(ZSTD_LIBRARY ${INSTALL_LIBDIR_64}/libzstd.a)
 else()
   set(ZSTD_LIBRARY ${INSTALL_LIBDIR}/libzstd.a)
@@ -394,7 +395,7 @@ else()
   set(LIB_FMT libfmt.a)
 endif()
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
   set(FMT_LIBRARY ${INSTALL_LIBDIR_64}/${LIB_FMT})
 else()
   set(FMT_LIBRARY ${INSTALL_LIBDIR}/${LIB_FMT})
@@ -432,7 +433,7 @@ ExternalProject_Add(lz4
   make -j${CPU_CORE}
 )
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
   set(LZ4_LIBRARY ${INSTALL_LIBDIR_64}/liblz4.a)
 else()
   set(LZ4_LIBRARY ${INSTALL_LIBDIR}/liblz4.a)
@@ -718,7 +719,7 @@ if (USE_PIKA_TOOLS)
   set(BZ2_LIBRARY ${INSTALL_LIBDIR}/libbz2.a)
 endif()
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
   set(ROCKSDB_LIBRARY ${INSTALL_LIBDIR_64}/librocksdb.a)
 else()
   set(ROCKSDB_LIBRARY ${INSTALL_LIBDIR}/librocksdb.a)

ci/release-build.sh

@@ -9,6 +9,12 @@ function install_deps() {
   elif [[ $OS == *"ubuntu"* ]]; then
     sudo apt-get install -y autoconf libprotobuf-dev protobuf-compiler
     sudo apt-get install -y clang-tidy-12
+  elif [[ $OS == *"rocky"* ]]; then
+    sudo dnf update -y
+    sudo dnf install -y bash cmake
+    sudo dnf install -y wget git autoconf gcc perl-Digest-SHA
+    sudo dnf install -y tcl which tar g++ tar epel-release gcc-c++ libstdc++-devel
+    sudo dnf install -y gcc-toolset-13
   else
     echo "not support $OS"
   fi
@@ -22,6 +28,9 @@ function configure_cmake() {
     cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE
   elif [[ $OS == *"ubuntu"* ]]; then
     cmake -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS="-s" -DCMAKE_EXE_LINKER_FLAGS="-s"
+  elif [[ $OS == *"rocky"* ]]; then
+    source /opt/rh/gcc-toolset-13/enable
+    cmake -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
   fi
   echo "configure cmake after ..."
 }

codis/Makefile

@@ -12,25 +12,25 @@ codis-deps:
 
 codis-dashboard: codis-deps
 	$(info build codis-dashboard)
-	@cd ${PRJ_ROOT}/cmd/dashboard && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-dashboard .
+	@cd ${PRJ_ROOT}/cmd/dashboard && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-dashboard .
 	@${PRJ_ROOT}/bin/codis-dashboard --default-config > ${PRJ_ROOT}/config/dashboard.toml
 
 codis-proxy: codis-deps
 	$(info build codis-proxy)
-	@cd ${PRJ_ROOT}/cmd/proxy && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-proxy .
+	@cd ${PRJ_ROOT}/cmd/proxy && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-proxy .
 	@${PRJ_ROOT}/bin/codis-proxy --default-config > ${PRJ_ROOT}/config/proxy.toml
 
 codis-admin: codis-deps
 	$(info build codis-admin)
-	@cd ${PRJ_ROOT}/cmd/admin && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-admin .
+	@cd ${PRJ_ROOT}/cmd/admin && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-admin .
 
 codis-ha: codis-deps
 	$(info build codis-ha)
-	@cd ${PRJ_ROOT}/cmd/ha && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-ha .
+	@cd ${PRJ_ROOT}/cmd/ha && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-ha .
 
 codis-fe: codis-deps
 	$(info build codis-fe)
-	@cd ${PRJ_ROOT}/cmd/fe && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-fe .
+	@cd ${PRJ_ROOT}/cmd/fe && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-fe .
 	@rm -rf ${PRJ_ROOT}/bin/assets && cp -rf ${PRJ_ROOT}/cmd/fe/assets ./bin/
 
 clean:

conf/pika.conf

@@ -18,11 +18,16 @@ rocksdb-periodic-second : 86400 * 3;
 # Master's run-id
 # master-run-id :
 
-# The number of threads for running Pika.
+# The number of Net-worker threads in Pika.
 # It's not recommended to set this value exceeds
 # the number of CPU cores on the deployment server.
 thread-num : 1
 
+# use Net worker thread to read redis Cache for [Get, HGet] command,
+# which can significantly improve QPS and reduce latency when cache hit rate is high
+# default value is "yes", set it to "no" if you wanna disable it
+rtc-cache-read : yes
+
 # Size of the thread pool, The threads within this pool
 # are dedicated to handling user requests.
 thread-pool-size : 12

include/pika_client_conn.h

@@ -19,6 +19,7 @@ struct TimeStat {
   void Reset() {
     enqueue_ts_ = dequeue_ts_ = 0;
     process_done_ts_ = 0;
+    before_queue_ts_ = 0;
   }
 
   uint64_t start_ts() const {
@@ -37,8 +38,13 @@ struct TimeStat {
     return process_done_ts_ > dequeue_ts_ ? process_done_ts_ - dequeue_ts_ : 0;
   }
 
+  uint64_t before_queue_time() const {
+    return process_done_ts_ > dequeue_ts_ ? before_queue_ts_ - enqueue_ts_ : 0;
+  }
+
   uint64_t enqueue_ts_;
   uint64_t dequeue_ts_;
+  uint64_t before_queue_ts_;
   uint64_t process_done_ts_;
 };
 
@@ -53,6 +59,7 @@ class PikaClientConn : public net::RedisConn {
     std::shared_ptr<std::string> resp_ptr;
     LogOffset offset;
     std::string db_name;
+    bool cache_miss_in_rtc_;
   };
 
   struct TxnStateBitMask {
@@ -67,9 +74,12 @@ class PikaClientConn : public net::RedisConn {
                  const net::HandleType& handle_type, int max_conn_rbuf_size);
   ~PikaClientConn() = default;
 
+  bool IsInterceptedByRTC(std::string& opt);
+
   void ProcessRedisCmds(const std::vector<net::RedisCmdArgsType>& argvs, bool async, std::string* response) override;
 
-  void BatchExecRedisCmd(const std::vector<net::RedisCmdArgsType>& argvs);
+  bool ReadCmdInCache(const net::RedisCmdArgsType& argv, const std::string& opt);
+  void BatchExecRedisCmd(const std::vector<net::RedisCmdArgsType>& argvs, bool cache_miss_in_rtc);
   int DealMessage(const net::RedisCmdArgsType& argv, std::string* response) override { return 0; }
   static void DoBackgroundTask(void* arg);
 
@@ -99,8 +109,7 @@ class PikaClientConn : public net::RedisConn {
   void AddKeysToWatch(const std::vector<std::string>& db_keys);
   void RemoveWatchedKeys();
   void SetTxnFailedFromKeys(const std::vector<std::string>& db_keys);
-  void SetAllTxnFailed();
-  void SetTxnFailedFromDBs(std::string db_name);
+  void SetTxnFailedIfKeyExists(const std::string target_db_name = "");
   void ExitTxn();
   bool IsInTxn();
   bool IsTxnInitFailed();
@@ -128,12 +137,12 @@ class PikaClientConn : public net::RedisConn {
   std::shared_ptr<User> user_;
 
   std::shared_ptr<Cmd> DoCmd(const PikaCmdArgsType& argv, const std::string& opt,
-                             const std::shared_ptr<std::string>& resp_ptr);
+                             const std::shared_ptr<std::string>& resp_ptr, bool cache_miss_in_rtc);
 
   void ProcessSlowlog(const PikaCmdArgsType& argv, uint64_t do_duration);
   void ProcessMonitor(const PikaCmdArgsType& argv);
 
-  void ExecRedisCmd(const PikaCmdArgsType& argv, std::shared_ptr<std::string>& resp_ptr);
+  void ExecRedisCmd(const PikaCmdArgsType& argv, std::shared_ptr<std::string>& resp_ptr, bool cache_miss_in_rtc);
   void TryWriteResp();
 };
 

include/pika_command.h

@@ -247,6 +247,7 @@ const std::string kCmdNameXInfo = "xinfo";
 
 const std::string kClusterPrefix = "pkcluster";
 
+
 /*
  * If a type holds a key, a new data structure
  * that uses the key will use this error
@@ -289,7 +290,7 @@ enum CmdFlags {
   kCmdFlagsOperateKey = (1 << 19),  // redis keySpace
   kCmdFlagsStream = (1 << 20),
   kCmdFlagsFast = (1 << 21),
-  kCmdFlagsSlow = (1 << 22),
+  kCmdFlagsSlow = (1 << 22)
 };
 
 void inline RedisAppendContent(std::string& str, const std::string& value);
@@ -536,6 +537,7 @@ class Cmd : public std::enable_shared_from_this<Cmd> {
   bool hasFlag(uint32_t flag) const;
   bool is_read() const;
   bool is_write() const;
+  bool isCacheRead() const;
 
   bool IsLocal() const;
   bool IsSuspend() const;
@@ -573,12 +575,16 @@ class Cmd : public std::enable_shared_from_this<Cmd> {
   uint32_t GetCmdId() const { return cmdId_; };
   bool CheckArg(uint64_t num) const;
 
+  bool IsCacheMissedInRtc() const;
+  void SetCacheMissedInRtc(bool value);
+
  protected:
   // enable copy, used default copy
   // Cmd(const Cmd&);
   void ProcessCommand(const HintKeys& hint_key = HintKeys());
   void InternalProcessCommand(const HintKeys& hint_key);
   void DoCommand(const HintKeys& hint_key);
+  bool DoReadCommandInCache();
   void LogCommand() const;
 
   std::string name_;
@@ -600,6 +606,7 @@ class Cmd : public std::enable_shared_from_this<Cmd> {
   uint64_t do_duration_ = 0;
   uint32_t cmdId_ = 0;
   uint32_t aclCategory_ = 0;
+  bool cache_missed_in_rtc_{false};
 
  private:
   virtual void DoInitial() = 0;

include/pika_conf.h

@@ -468,6 +468,7 @@ class PikaConf : public pstd::BaseConf {
 
   // Immutable config items, we don't use lock.
   bool daemonize() { return daemonize_; }
+  bool rtc_cache_read_enabled() { return rtc_cache_read_enabled_; }
   std::string pidfile() { return pidfile_; }
   int binlog_file_size() { return binlog_file_size_; }
   std::vector<rocksdb::CompressionType> compression_per_level();
@@ -930,6 +931,7 @@ class PikaConf : public pstd::BaseConf {
   int level0_file_num_compaction_trigger_ = 4;
   int64_t max_client_response_size_ = 0;
   bool daemonize_ = false;
+  bool rtc_cache_read_enabled_ = false;
   int timeout_ = 0;
   std::string server_id_;
   std::string run_id_;