Merge remote-tracking branch 'origin/java_benchmarks' into java_bench…

…marks-add-json Signed-off-by: Yury-Fridlyand <[email protected]>
Bit-Quill · Sep 26, 2023 · 9638f15 · 9638f15
2 parents 94441e2 + 8764730
commit 9638f15
Show file tree

Hide file tree

Showing 90 changed files with 3,244 additions and 1,657 deletions.
diff --git a/.github/workflows/install-redis/action.yml b/.github/workflows/install-redis/action.yml
@@ -16,7 +16,7 @@ runs:
         - run: mkdir ~/redis-binaries
           shell: bash
 
-        - uses: actions/checkout@v3
+        - uses: actions/checkout@v4
           with:
               submodules: recursive
 

diff --git a/.github/workflows/lint-rust/action.yml b/.github/workflows/lint-rust/action.yml
@@ -10,38 +10,31 @@ runs:
     using: "composite"
 
     steps:
-        - uses: actions/checkout@v3
+        - uses: actions/checkout@v4
           with:
               submodules: recursive
-
-        - uses: actions-rs/toolchain@v1
-          with:
-              profile: minimal
-              toolchain: stable
-              override: true
-              components: rustfmt, clippy
 
-        - uses: Swatinem/rust-cache@v1
-
+        - uses: dtolnay/rust-toolchain@stable
+        - uses: Swatinem/rust-cache@v2
+
         - run: |
-            sudo apt update
-            sudo apt install protobuf-compiler
+              sudo apt update
+              sudo apt install protobuf-compiler
+          shell: bash
+
+        - run: cargo fmt --all -- --check
+          working-directory: ${{ inputs.cargo-toml-folder }}
+          shell: bash
+
+        - run: cargo clippy --all-features --all-targets -- -D warnings
+          working-directory: ${{ inputs.cargo-toml-folder }}
           shell: bash
-        - uses: actions-rs/cargo@v1
-          with:
-              command: fmt
-              args: --manifest-path ${{ inputs.cargo-toml-folder }}/Cargo.toml --all -- --check
 
-        - uses: actions-rs/cargo@v1
-          with:
-              command: clippy
-              args: --manifest-path ${{ inputs.cargo-toml-folder }}/Cargo.toml --all-features --all-targets -- -D warnings
-
         - run: |
-              cd ${{ inputs.cargo-toml-folder }}
               cargo update
               cargo install cargo-deny
               cargo deny check licenses --config ${GITHUB_WORKSPACE}/deny.toml
+          working-directory: ${{ inputs.cargo-toml-folder }}
           shell: bash
 
         - name: doc

diff --git a/.github/workflows/lint-ts.yml b/.github/workflows/lint-ts.yml
@@ -5,10 +5,12 @@ on:
         paths:
             - benchmarks/node/**
             - node/**
+            - benchmarks/utilities/*
     pull_request:
         paths:
             - benchmarks/node/**
             - node/**
+            - benchmarks/utilities/*
 
 env:
     CARGO_TERM_COLOR: always
@@ -17,7 +19,7 @@ jobs:
         runs-on: ubuntu-latest
         timeout-minutes: 10
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
             - name: lint ts
               run: |
                   npm install @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-tsdoc eslint typescript

diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml
@@ -28,7 +28,7 @@ jobs:
                     - 7.2.0
 
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
@@ -67,10 +67,11 @@ jobs:
         timeout-minutes: 15
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
             - uses: ./.github/workflows/lint-rust
               with:
                   cargo-toml-folder: ./node/rust-client
+              name: lint node rust
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -28,7 +28,7 @@ jobs:
                     - 7.2.0
 
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
@@ -51,7 +51,7 @@ jobs:
               working-directory: ./python
               run: |
                   python -m pip install --upgrade pip
-                  pip install flake8 isort black
+                  pip install flake8 isort black mypy-protobuf
 
             - name: Lint with isort
               working-directory: ./python
@@ -78,25 +78,43 @@ jobs:
             - name: Generate protobuf files
               working-directory: .
               run: |
-                  protoc -Iprotobuf=./babushka-core/src/protobuf/ --python_out=./python/python/pybushka ./babushka-core/src/protobuf/*.proto
+                  MYPY_PROTOC_PATH=`which protoc-gen-mypy`
+                  protoc --plugin=protoc-gen-mypy=${MYPY_PROTOC_PATH} -Iprotobuf=./babushka-core/src/protobuf/ --python_out=./python/python/pybushka --mypy_out=./python/python/pybushka ./babushka-core/src/protobuf/*.proto
 
-            - name: Test with pytest
+            - name: Build Pybushka
               working-directory: ./python
               run: |
                   python -m venv .env
                   source .env/bin/activate
                   pip install -r requirements.txt
                   maturin develop --release
+
+            - name: Type check with mypy
+              working-directory: ./python
+              run: |
+                  # The type check should run inside the virtual env to get
+                  # all installed dependencies and build files
+                  source .env/bin/activate
+                  pip install mypy types-protobuf
+                  # Install the benchmark requirements 
+                  pip install -r ../benchmarks/python/requirements.txt
+                  python -m mypy ..
+              
+            - name: Test with pytest
+              working-directory: ./python
+              run: |
+                  source .env/bin/activate
                   pytest --asyncio-mode=auto
 
     lint-rust:
         runs-on: ubuntu-latest
         timeout-minutes: 15
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
             - uses: ./.github/workflows/lint-rust
               with:
-                  cargo-toml-folder: ./python
+                  cargo-toml-folder: ./python                  
+              name: lint python-rust
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -28,7 +28,7 @@ jobs:
                     - 7.2.0
 
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
@@ -42,14 +42,8 @@ jobs:
                   sudo apt update
                   sudo apt install protobuf-compiler
 
-            - name: Install toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  toolchain: stable
-                  override: true
-                  components: rustfmt
-
-            - uses: Swatinem/rust-cache@v1
+            - uses: dtolnay/rust-toolchain@stable
+            - uses: Swatinem/rust-cache@v2
 
             - name: Run tests
               working-directory: ./babushka-core
@@ -69,18 +63,21 @@ jobs:
         runs-on: ubuntu-latest
         timeout-minutes: 20
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
             - uses: ./.github/workflows/lint-rust
               with:
                   cargo-toml-folder: ./babushka-core
+              name: lint babushka-core
 
             - uses: ./.github/workflows/lint-rust
               with:
                   cargo-toml-folder: ./logger_core
+              name: lint logger
 
             - uses: ./.github/workflows/lint-rust
               with:
                   cargo-toml-folder: ./benchmarks/rust
+              name: lint benchmark
diff --git a/README.md b/README.md
@@ -23,9 +23,9 @@ Babushka (temporary name, final name pending) is a collection of open source Red
 
 -   GCC
 -   pkg-config
--   protobuf-compiler (protoc)
+-   protobuf-compiler (protoc) >= V3
 -   openssl
--   libssl-dev
+-   libssl-dev // for amazon-linux install openssl-devel
 -   python3
 
 Installation for ubuntu:

diff --git a/babushka-core/.cargo/config.toml b/babushka-core/.cargo/config.toml
@@ -0,0 +1,3 @@
+[env]
+BABUSHKA_NAME = "Babushka" # This should be overwritten by each wrapper library.
+BABUSHKA_VERSION = "0.1.0"
diff --git a/babushka-core/Cargo.toml b/babushka-core/Cargo.toml
@@ -15,12 +15,12 @@ signal-hook = "^0.3"
 signal-hook-tokio = {version = "^0.3", features = ["futures-v0_3"] }
 tokio = { version = "1", features = ["macros", "time"] }
 logger_core = {path = "../logger_core"}
-dispose = "0.4.0"
+dispose = "0.5.0"
 tokio-util = {version = "^0.7", features = ["rt"]}
 num_cpus = "^1.15"
 tokio-retry = "0.3.0"
 protobuf = {version= "3", features = ["bytes", "with-bytes"]}
-integer-encoding = "3.0.4"
+integer-encoding = "4.0.0"
 thiserror = "1"
 rand = "0.8.5"
 futures-intrusive = "0.5.0"
@@ -35,7 +35,7 @@ criterion = { version = "^0.5", features = ["html_reports", "async_tokio"] }
 which = "4.4.0"
 ctor = "0.2.2"
 redis = { path = "../submodules/redis-rs/redis", features = ["tls-rustls-insecure"] }
-iai-callgrind = "0.3.1"
+iai-callgrind = "0.6"
 tokio = { version = "1", features = ["rt-multi-thread"] }
 once_cell = "1.18.0"
 

diff --git a/babushka-core/src/client/mod.rs b/babushka-core/src/client/mod.rs
@@ -1,18 +1,18 @@
 use crate::connection_request::{
     AddressInfo, AuthenticationInfo, ConnectionRequest, ReadFromReplicaStrategy, TlsMode,
 };
-pub use client_cmd::ClientCMD;
 use futures::FutureExt;
 use redis::cluster_async::ClusterConnection;
 use redis::cluster_routing::{RoutingInfo, SingleNodeRoutingInfo};
 use redis::{
     aio::{ConnectionLike, ConnectionManager, MultiplexedConnection},
     RedisResult,
 };
+pub use standalone_client::StandaloneClient;
 use std::io;
 use std::time::Duration;
-mod client_cmd;
 mod reconnecting_connection;
+mod standalone_client;
 
 pub const HEARTBEAT_SLEEP_DURATION: Duration = Duration::from_secs(1);
 
@@ -78,8 +78,8 @@ pub(super) fn get_connection_info(
 
 #[derive(Clone)]
 pub enum ClientWrapper {
-    CMD(ClientCMD),
-    CME {
+    Standalone(StandaloneClient),
+    Cluster {
         client: ClusterConnection,
         read_from_replicas: bool,
     },
@@ -109,9 +109,9 @@ impl Client {
     ) -> redis::RedisFuture<'a, redis::Value> {
         run_with_timeout(self.response_timeout, async {
             match self.internal_client {
-                ClientWrapper::CMD(ref mut client) => client.send_packed_command(cmd).await,
+                ClientWrapper::Standalone(ref mut client) => client.send_packed_command(cmd).await,
 
-                ClientWrapper::CME {
+                ClientWrapper::Cluster {
                     ref mut client,
                     read_from_replicas,
                 } => {
@@ -133,11 +133,11 @@ impl Client {
     ) -> redis::RedisFuture<'a, Vec<redis::Value>> {
         (async move {
             match self.internal_client {
-                ClientWrapper::CMD(ref mut client) => {
+                ClientWrapper::Standalone(ref mut client) => {
                     client.send_packed_commands(cmd, offset, count).await
                 }
 
-                ClientWrapper::CME {
+                ClientWrapper::Cluster {
                     ref mut client,
                     read_from_replicas: _,
                 } => {
@@ -205,16 +205,16 @@ async fn create_cluster_client(
 
 #[derive(thiserror::Error)]
 pub enum ConnectionError {
-    CMD(client_cmd::ClientCMDConnectionError),
-    CME(redis::RedisError),
+    Standalone(standalone_client::StandaloneClientConnectionError),
+    Cluster(redis::RedisError),
     Timeout,
 }
 
 impl std::fmt::Debug for ConnectionError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            Self::CMD(arg0) => f.debug_tuple("CMD").field(arg0).finish(),
-            Self::CME(arg0) => f.debug_tuple("CME").field(arg0).finish(),
+            Self::Standalone(arg0) => f.debug_tuple("Standalone").field(arg0).finish(),
+            Self::Cluster(arg0) => f.debug_tuple("Cluster").field(arg0).finish(),
             Self::Timeout => write!(f, "Timeout"),
         }
     }
@@ -223,8 +223,8 @@ impl std::fmt::Debug for ConnectionError {
 impl std::fmt::Display for ConnectionError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            ConnectionError::CMD(err) => write!(f, "{err:?}"),
-            ConnectionError::CME(err) => write!(f, "{err}"),
+            ConnectionError::Standalone(err) => write!(f, "{err:?}"),
+            ConnectionError::Cluster(err) => write!(f, "{err}"),
             ConnectionError::Timeout => f.write_str("connection attempt timed out"),
         }
     }
@@ -243,16 +243,16 @@ impl Client {
             let internal_client = if request.cluster_mode_enabled {
                 let (client, read_from_replicas) = create_cluster_client(request)
                     .await
-                    .map_err(ConnectionError::CME)?;
-                ClientWrapper::CME {
+                    .map_err(ConnectionError::Cluster)?;
+                ClientWrapper::Cluster {
                     client,
                     read_from_replicas,
                 }
             } else {
-                ClientWrapper::CMD(
-                    ClientCMD::create_client(request)
+                ClientWrapper::Standalone(
+                    StandaloneClient::create_client(request)
                         .await
-                        .map_err(ConnectionError::CMD)?,
+                        .map_err(ConnectionError::Standalone)?,
                 )
             };