Update scripts and reporing.

Signed-off-by: Yury-Fridlyand <[email protected]>

benchmarks/install_and_test.sh

@@ -26,9 +26,9 @@ runNode=0
 runCsharp=0
 runJava=0
 runRust=0
-concurrentTasks="1 10 100 1000"
-dataSize="100 4000"
-clientCount="1"
+concurrentTasks="10 100 1000"
+dataSize="100"
+clientCount="2"
 chosenClients="all"
 host="localhost"
 port=6379
@@ -71,18 +71,20 @@ function runCSharpBenchmark(){
 }
 
 function runJavaBenchmark(){
+  cd ${BENCH_FOLDER}/../kotlin
+  ./gradlew build
   cd ${BENCH_FOLDER}/../java
-  echo "./gradlew run --args=\"--resultsFile=${BENCH_FOLDER}/$1 --clients $chosenClients --host $host --port $port\""
-#  ./gradlew run --args="--resultsFile=../$1 --dataSize $2 --concurrentTasks $concurrentTasks --clients $chosenClients --host $host --port $port --clientCount $clientCount $tlsFlag"
-  ./gradlew run --args="--resultsFile=${BENCH_FOLDER}/$1 --clients $chosenClients --host $host --port $port"
-  cd ${BENCH_FOLDER}/java
+  cargo build
+  # ./gradlew build
+  # echo ./gradlew :benchmarks:run --args="-resultsFile ${BENCH_FOLDER}/$1 -dataSize \"$2\" -concurrentTasks \"$concurrentTasks\" -clients \"$chosenClients\" -host $host -port $port -clientCount \"$clientCount\" $tlsFlag"
+  ./gradlew :benchmarks:run --args="-resultsFile ${BENCH_FOLDER}/$1 -dataSize \"$2\" -concurrentTasks \"$concurrentTasks\" -clients \"$chosenClients\" -host $host -port $port -clientCount \"$clientCount\" $tlsFlag"
 }
 
 function runRustBenchmark(){
   rustConcurrentTasks=
   for value in $concurrentTasks
   do
-    rustConcurrentTasks=$rustConcurrentTasks" --concurrentTasks "$value
+    rustConcurrentTasks="$rustConcurrentTasks --concurrentTasks $value"
   done
   cd ${BENCH_FOLDER}/rust
   cargo run --release -- --resultsFile=../$1 --dataSize $2 $rustConcurrentTasks --host $host --clientCount $clientCount $tlsFlag $clusterFlag $portFlag
@@ -198,7 +200,7 @@ do
         -java)
             runAllBenchmarks=0
             runJava=1
-            chosenClients="Babushka"
+            chosenClients="Lettuce,JNI_FFI,JNA_FFI,Kotlin"
             ;;
         -lettuce)
             runAllBenchmarks=0

benchmarks/utilities/csv_exporter.py

@@ -50,6 +50,3 @@
                 json_object["language"] = language
                 values = [json_object[field] for field in base_fields]
                 writer.writerow(values)
-
-for json_file_full_path in sys.argv[1:-1]:
-    os.remove(json_file_full_path)

java/benchmarks/src/main/java/javababushka/benchmarks/BenchmarkingApp.java

@@ -198,7 +198,7 @@ private static int[] parseIntListOption(String line) throws ParseException {
       lineValue = lineValue.substring(1, lineValue.length() - 1);
     }
     // check if it's the correct format
-    if (!lineValue.matches("\\d+(\\s+\\d+)?")) {
+    if (!lineValue.matches("\\d+(\\s+\\d+)*")) {
       throw new ParseException("Invalid option: " + line);
     }
     // split the string into a list of integers

java/benchmarks/src/main/java/javababushka/benchmarks/utils/Benchmarking.java

@@ -89,32 +89,38 @@ public static Map<ChosenAction, LatencyResults> calculateResults(
       ChosenAction action = entry.getKey();
       ArrayList<Long> latencies = entry.getValue();
 
-      double avgLatency = latencies.stream().mapToLong(Long::longValue).sum() / latencies.size();
+      double avgLatency =
+          latencies.stream().mapToLong(Long::longValue).sum() * 1e-6 / latencies.size();
 
       Collections.sort(latencies);
       results.put(
           action,
           new LatencyResults(
               avgLatency,
-              percentile(latencies, 50),
-              percentile(latencies, 90),
-              percentile(latencies, 99),
-              stdDeviation(latencies, avgLatency)));
+              percentile(latencies, 50) * 1e-6,
+              percentile(latencies, 90) * 1e-6,
+              percentile(latencies, 99) * 1e-6,
+              stdDeviation(latencies, avgLatency) * 1e-6));
     }
 
     return results;
   }
 
-  public static void printResults(Map<ChosenAction, LatencyResults> resultsMap) {
+  public static void printResults(
+      Map<ChosenAction, LatencyResults> resultsMap, double duration, int iterations) {
+    System.out.printf("Runtime s: %f%n", duration);
+    System.out.printf("Iterations: %d%n", iterations);
+    System.out.printf("TPS: %f%n", iterations / duration);
     for (Map.Entry<ChosenAction, LatencyResults> entry : resultsMap.entrySet()) {
       ChosenAction action = entry.getKey();
       LatencyResults results = entry.getValue();
 
-      System.out.println("Avg. time in ms per " + action + ": " + results.avgLatency / 1000000);
-      System.out.println(action + " p50 latency in ms: " + results.p50Latency / 1000000);
-      System.out.println(action + " p90 latency in ms: " + results.p90Latency / 1000000);
-      System.out.println(action + " p99 latency in ms: " + results.p99Latency / 1000000);
-      System.out.println(action + " std dev in ms: " + results.stdDeviation / 1000000);
+      System.out.printf("===> %s <===%n", action);
+      System.out.printf("avg. time ms: %f%n", results.avgLatency);
+      System.out.printf("std dev ms: %f%n", results.stdDeviation);
+      System.out.printf("p50 latency ms: %f%n", results.p50Latency);
+      System.out.printf("p90 latency ms: %f%n", results.p90Latency);
+      System.out.printf("p99 latency ms: %f%n", results.p99Latency);
     }
   }
 
@@ -188,6 +194,7 @@ public static void testClientSetGet(
                       e.printStackTrace();
                     }
                   });
+          long ended = System.nanoTime();
 
           var calculatedResults = calculateResults(actionResults);
           if (config.resultsFile.isPresent()) {
@@ -198,9 +205,9 @@ public static void testClientSetGet(
                 clientCreator.get().getName(),
                 clientCount,
                 concurrentNum,
-                iterationCounter.get() * 1e9 / (System.nanoTime() - started));
+                iterationCounter.get() * 1e9 / (ended - started));
           }
-          printResults(calculatedResults);
+          printResults(calculatedResults, (ended - started) / 1e9, iterationCounter.get());
         }
       }
     }

java/benchmarks/src/main/java/javababushka/benchmarks/utils/JsonWriter.java

@@ -38,7 +38,7 @@ public static void Write(
               // TODO:  is_cluster
               .data_size(dataSize)
               .client(client)
-              .client_count(clientCount)
+              .clientCount(clientCount)
               .num_of_tasks(numOfTasks)
               .tps(tps)
               .get_existing_average_latency(
@@ -79,7 +79,7 @@ public static void Write(
   @Builder
   public static class Measurements {
     private String client;
-    private int client_count;
+    private int clientCount;
     private int data_size;
     private double get_existing_average_latency;
     private double get_existing_p50_latency;

java/benchmarks/src/test/java/javababushka/benchmarks/jedis/JedisClientIT.java

@@ -66,6 +66,6 @@ public void testResourceSetGet() {
       latencies.get(latency.getKey()).add(latency.getValue());
     }
 
-    Benchmarking.printResults(Benchmarking.calculateResults(latencies));
+    Benchmarking.printResults(Benchmarking.calculateResults(latencies), 0, iterations);
   }
 }

java/javababushka.hpp

@@ -4,9 +4,6 @@
 #include <ostream>
 #include <new>
 
-template<typename T = void>
-struct Option;
-
 struct BabushkaResultStr {
   const char *error;
   const char *result;
@@ -19,12 +16,6 @@ struct BabushkaResult {
   int64_t num;
 };
 
-struct BabushkaClient {
-  Option<Mutex<Runtime>> runtime;
-  Option<Mutex<MultiplexedConnection>> connection;
-  int32_t data;
-};
-
 extern "C" {
 
 BabushkaResultStr static_function_which_throws();
@@ -53,8 +44,4 @@ BabushkaResult set0(uint64_t ptr, const char *key, const char *value);
 
 BabushkaResult get0(uint64_t ptr, const char *key);
 
-BabushkaResult set(BabushkaClient *self, const char *key, const char *value);
-
-BabushkaResult get(BabushkaClient *self, const char *key);
-
 } // extern "C"