Go client: Adding a skeleton to create client connection and invoke S…

…ET and GET commands. (valkey-io#1937) * Go: create client connection (valkey-io#1303) * Go: Implement benchmarking (valkey-io#1330) * Update DEVELOPER.md with instructions to extend LD_LIBRARY_PATH on ubuntu/centos * Go: Fix bug with -clients=all setting (valkey-io#1339) * Go: Fixes a bug where the benchmarks were ran twice for go-redis when the -clients option was set to "all" * Go: update protobuf version to address dependabot alert (valkey-io#1352) * Go client: Preparing the skeleton code to create the client connection and invoke set and get commands Signed-off-by: Janhavi Gupta <[email protected]> --------- Signed-off-by: Janhavi Gupta <[email protected]> Co-authored-by: Andrew Carbonetto <[email protected]> Co-authored-by: Aaron <[email protected]> Co-authored-by: aaron-congo <[email protected]>
Bit-Quill · Aug 19, 2024 · 322d8d7 · 322d8d7
1 parent 0f00fd6
commit 322d8d7
Show file tree

Hide file tree

Showing 30 changed files with 2,193 additions and 93 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -106,7 +106,22 @@ jobs:
 
             - name: Run tests
               working-directory: ./go
-              run: make test
+              run: |
+                make test
+
+            - uses: ./.github/workflows/test-benchmark
+              with:
+                  language-flag: -go
+
+            - name: Upload logs and reports
+              if: always()
+              continue-on-error: true
+              uses: actions/upload-artifact@v4
+              with:
+                  name: reports-go-${{ matrix.go }}-redis-${{ matrix.redis }}-${{ matrix.os }}
+                  path: |
+                      utils/clusters/**
+                      benchmarks/results/**
 
     build-amazonlinux-latest:
         if: github.repository_owner == 'valkey-io'
@@ -158,6 +173,9 @@ jobs:
               working-directory: ./go
               run: make install-tools-go${{ matrix.go }}
 
+            - name: Set LD_LIBRARY_PATH
+              run: echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/go/target/release/deps/" >> $GITHUB_ENV
+
             - name: Build client
               working-directory: ./go
               run: make build
@@ -168,7 +186,17 @@ jobs:
 
             - name: Run tests
               working-directory: ./go
-              run: make test
+              run: |
+                make test
+
+            - name: Upload cluster manager logs
+              if: always()
+              continue-on-error: true
+              uses: actions/upload-artifact@v4
+              with:
+                  name: cluster-manager-logs-${{ matrix.go }}-redis-6-amazonlinux
+                  path: |
+                      utils/clusters/**
 
     lint-rust:
         timeout-minutes: 15

diff --git a/benchmarks/install_and_test.sh b/benchmarks/install_and_test.sh
@@ -25,6 +25,7 @@ runPython=0
 runNode=0
 runCsharp=0
 runJava=0
+runGo=0
 runRust=0
 concurrentTasks="1 10 100 1000"
 dataSize="100 4000"
@@ -76,6 +77,12 @@ function runJavaBenchmark(){
   ./gradlew :benchmarks:run --args="-resultsFile \"${BENCH_FOLDER}/$1\" --dataSize \"$2\" --concurrentTasks \"$concurrentTasks\" --clients \"$chosenClients\" --host $host $portFlag --clientCount \"$clientCount\" $tlsFlag $clusterFlag $minimalFlag"
 }
 
+function runGoBenchmark(){
+    cd ${BENCH_FOLDER}/../go/benchmarks
+    export LD_LIBRARY_PATH=${BENCH_FOLDER}/../go/target/release:$LD_LIBRARY_PATH
+    go run . -resultsFile ${BENCH_FOLDER}/$1 -dataSize $2 -concurrentTasks $concurrentTasks -clients $chosenClients -host $host $portFlag -clientCount $clientCount $tlsFlag $clusterFlag $minimalFlag
+}
+
 function runRustBenchmark(){
   rustConcurrentTasks=
   for value in $concurrentTasks
@@ -115,7 +122,7 @@ function resultFileName() {
 
 function Help() {
     echo Running the script without any arguments runs all benchmarks.
-    echo Pass -node, -csharp, -python, -java as arguments in order to run the node, csharp, python, or java benchmarks accordingly.
+    echo Pass -node, -csharp, -python, -java, -go as arguments in order to run the node, csharp, python, java, or go benchmarks accordingly.
     echo Multiple such flags can be passed.
     echo Pass -no-csv to skip analysis of the results.
     echo
@@ -204,6 +211,15 @@ do
             runJava=1
             chosenClients="Jedis"
             ;;
+        -go)
+            runAllBenchmarks=0
+            runGo=1
+            ;;
+        -go-redis)
+            runAllBenchmarks=0
+            runGo=1
+            chosenClients="go-redis"
+            ;;
         -csharp)
             runAllBenchmarks=0
             runCsharp=1
@@ -274,6 +290,13 @@ do
         runJavaBenchmark $javaResults $currentDataSize
     fi
 
+    if [ $runAllBenchmarks == 1 ] || [ $runGo == 1 ];
+    then
+        goResults=$(resultFileName go $currentDataSize)
+        resultFiles+=$goResults" "
+        runGoBenchmark $goResults $currentDataSize
+    fi
+
     if [ $runAllBenchmarks == 1 ] || [ $runRust == 1 ];
     then
         rustResults=$(resultFileName rust $currentDataSize)

diff --git a/benchmarks/utilities/csv_exporter.py b/benchmarks/utilities/csv_exporter.py
@@ -43,7 +43,7 @@
 
             json_file_name = os.path.basename(json_file_full_path)
 
-            languages = ["csharp", "node", "python", "rust", "java"]
+            languages = ["csharp", "node", "python", "rust", "java", "go"]
             language = next(
                 (language for language in languages if language in json_file_name), None
             )

diff --git a/go/.gitignore b/go/.gitignore
@@ -5,3 +5,7 @@ reports
 
 # cbindgen generated header file
 lib.h
+
+# benchmarking results
+benchmarks/results/**
+benchmarks/gobenchmarks.json
diff --git a/go/Cargo.toml b/go/Cargo.toml
@@ -13,6 +13,7 @@ redis = { path = "../submodules/redis-rs/redis", features = ["aio", "tokio-comp"
 glide-core = { path = "../glide-core", features = ["socket-layer"] }
 tokio = { version = "^1", features = ["rt", "macros", "rt-multi-thread", "time"] }
 protobuf = { version = "3.3.0", features = [] }
+derivative = "2.2.0"
 
 [profile.release]
 lto = true

diff --git a/go/DEVELOPER.md b/go/DEVELOPER.md
@@ -1,12 +1,12 @@
 # Developer Guide
 
-This document describes how to set up your development environment to build and test the GLIDE for Redis Go wrapper.
+This document describes how to set up your development environment to build and test the Valkey GLIDE Go wrapper.
 
 ### Development Overview
 
 We're excited to share that the GLIDE Go client is currently in development! However, it's important to note that this client is a work in progress and is not yet complete or fully tested. Your contributions and feedback are highly encouraged as we work towards refining and improving this implementation. Thank you for your interest and understanding as we continue to develop this Go wrapper.
 
-The GLIDE for Redis Go wrapper consists of both Go and Rust code. The Go and Rust components communicate in two ways:
+The Valkey GLIDE Go wrapper consists of both Go and Rust code. The Go and Rust components communicate in two ways:
 1. Using the [protobuf](https://github.com/protocolbuffers/protobuf) protocol.
 2. Using shared C objects. [cgo](https://pkg.go.dev/cmd/cgo) is used to interact with the C objects from Go code.
 
@@ -25,11 +25,10 @@ Software Dependencies
 -   openssl
 -   openssl-dev
 -   rustup
--   redis
 
-**Redis installation**
+**Valkey installation**
 
-To install redis-server and redis-cli on your host, follow the [Redis Installation Guide](https://redis.io/docs/install/install-redis/).
+To install valkey-server and valkey-cli on your host, follow the [Valkey Installation Guide](https://github.com/valkey-io/valkey).
 
 **Dependencies installation for Ubuntu**
 
@@ -102,7 +101,8 @@ Before starting this step, make sure you've installed all software requirements.
 
 1. Clone the repository:
     ```bash
-    git clone https://github.com/valkey-io/valkey-glide.git
+    VERSION=0.1.0 # You can modify this to other released version or set it to "main" to get the unstable branch
+    git clone --branch ${VERSION} https://github.com/valkey-io/valkey-glide.git
     cd valkey-glide
     ```
 2. Initialize git submodule:
@@ -114,20 +114,28 @@ Before starting this step, make sure you've installed all software requirements.
     cd go
     make install-build-tools
     ```
-4. Build the Go wrapper:
+4. If on CentOS or Ubuntu, add the glide-rs library to LD_LIBRARY_PATH:
+    ```bash
+    # Replace "<path to valkey-glide>" with the path to the valkey-glide root, eg "$HOME/Projects/valkey-glide"
+    GLIDE_ROOT_FOLDER_PATH=<path to valkey-glide>
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GLIDE_ROOT_FOLDER_PATH/go/target/release/deps/
+    ```
+5. Build the Go wrapper:
     ```bash
     make build
     ```
-5. Run tests:
-    1. Ensure that you have installed redis-server and redis-cli on your host. You can find the Redis installation guide at the following link: [Redis Installation Guide](https://redis.io/docs/install/install-redis/install-redis-on-linux/).
+6. Run tests:
+    1. Ensure that you have installed valkey-server and valkey-cli on your host. You can find the Valkey installation guide at the following link: [Valkey Installation Guide](https://github.com/valkey-io/valkey).
     2. Execute the following command from the go folder:
         ```bash
         go test -race ./...
         ```
-6. Install Go development tools with:
-
+7. Install Go development tools with:
     ```bash
+    # For go1.22:
     make install-dev-tools
+    # For go1.18:
+    make install-dev-tools-go1.18
     ```
 
 ### Test
@@ -191,7 +199,11 @@ Run from the main `/go` folder
 
 1. Go
     ```bash
+    # For go1.22:
     make install-dev-tools
+    # For go1.18:
+    make install-dev-tools-go1.18
+
     make lint
     ```
 2. Rust
@@ -211,6 +223,18 @@ Run from the main `/go` folder
 make format
 ```
 
+### Benchmarks
+
+To run the benchmarks, ensure you have followed the [build and installation steps](#building-and-installation-steps) (the tests do not have to be run). Then execute the following:
+
+```bash
+cd go/benchmarks
+# To see a list of available options and their defaults:
+go run . -help
+# An example command setting various options:
+go run . -resultsFile gobenchmarks.json -dataSize "100 1000" -concurrentTasks "10 100" -clients all -host localhost -port 6379 -clientCount "1 5" -tls
+```
+
 ### Recommended extensions for VS Code
 
 -   [Go](https://marketplace.visualstudio.com/items?itemName=golang.Go)

diff --git a/go/Makefile b/go/Makefile
@@ -1,19 +1,22 @@
 install-build-tools:
 	go install google.golang.org/protobuf/cmd/[email protected]
+	cargo install cbindgen
 
-install-dev-tools-go1.18.10:
+install-dev-tools-go1.18:
 	go install github.com/vakenbolt/[email protected]
 	go install mvdan.cc/[email protected]
 	go install github.com/segmentio/[email protected]
 	go install honnef.co/go/tools/cmd/[email protected]
-	cargo install cbindgen
 
-install-dev-tools-go1.22.0:
+install-dev-tools-go1.18.10: install-dev-tools-go1.18
+
+install-dev-tools-go1.22:
 	go install github.com/vakenbolt/[email protected]
 	go install mvdan.cc/[email protected]
 	go install github.com/segmentio/[email protected]
 	go install honnef.co/go/tools/cmd/[email protected]
-	cargo install cbindgen
+
+install-dev-tools-go1.22.0: install-dev-tools-go1.22
 
 install-dev-tools: install-dev-tools-go1.22.0
 
@@ -55,8 +58,12 @@ format:
 	golines -w --shorten-comments -m 127 .
 
 test:
+	LD_LIBRARY_PATH=$(shell find . -name libglide_rs.so|tail -1|xargs dirname|xargs readlink -f):${LD_LIBRARY_PATH} \
 	go test -v -race ./...
 
+# Note: this task is no longer run by CI because:
+# - build failures that occur while running the task can be hidden by the task; CI still reports success in these scenarios.
+# - there is not a good way to both generate a test report and log the test outcomes to GH actions.
 test-and-report:
 	mkdir -p reports
 	go test -v -race ./... -json | go-test-report -o reports/test-report.html