Unify parameters: second test run with log files

Beuth-Erdelt · Sep 11, 2024 · abe6658 · abe6658
1 parent 068aeff
commit abe6658
Show file tree

Hide file tree

Showing 34 changed files with 7,266 additions and 222 deletions.
diff --git a/benchbase.py b/benchbase.py
@@ -244,7 +244,7 @@
     if SF:
         experiment.workload['info'] = experiment.workload['info']+" Scaling factor (e.g., number of warehouses) is {}.".format(SF)
     if SD:
-        experiment.workload['info'] = experiment.workload['info']+" Benchmarking runs for {} minutes.".format(SD)
+        experiment.workload['info'] = experiment.workload['info']+" Benchmarking runs for {} minutes.".format(int(SD/60))
     if len(args.dbms):
         # import is limited to single DBMS
         experiment.workload['info'] = experiment.workload['info']+"\nBenchmark is limited to DBMS {}.".format(", ".join(args.dbms))

diff --git a/bexhoma/experiments.py b/bexhoma/experiments.py
@@ -2034,6 +2034,7 @@ def show_summary(self):
         #evaluate.load_experiment(code=code, silent=False)
         evaluation = evaluators.ycsb(code=code, path=resultfolder)
         #####################
+        test_loading = False
         df = evaluation.get_df_loading()
         if not df.empty:
             print("\n### Loading")
@@ -2046,6 +2047,7 @@ def show_summary(self):
             df_aggregated.sort_values(['experiment_run','target','pod_count'], inplace=True)
             df_aggregated_loaded = df_aggregated[['experiment_run',"threads","target","pod_count","[OVERALL].Throughput(ops/sec)","[OVERALL].RunTime(ms)","[INSERT].Return=OK","[INSERT].99thPercentileLatency(us)"]]
             print(df_aggregated_loaded)
+            test_loading = True
         #####################
         df = evaluation.get_df_benchmarking()
         if not df.empty:
@@ -2063,7 +2065,8 @@ def show_summary(self):
         #evaluation = evaluators.ycsb(code=code, path=path)
         #####################
         self.show_summary_monitoring()
-        evaluation.test_results_column(df_aggregated_loaded, "[OVERALL].Throughput(ops/sec)")
+        if test_loading:
+            evaluation.test_results_column(df_aggregated_loaded, "[OVERALL].Throughput(ops/sec)")
         evaluation.test_results_column(df_aggregated_reduced, "[OVERALL].Throughput(ops/sec)")
     def show_summary_monitoring(self):
         resultfolder = self.cluster.config['benchmarker']['resultfolder']

diff --git a/docs/TestCases.md b/docs/TestCases.md
diff --git a/logs_tests/test_benchbase_testcase_1.log b/logs_tests/test_benchbase_testcase_1.log
@@ -0,0 +1,124 @@
+Dashboard                     : is running
+Experiment                    : has code 1725988389
+Experiment                    : starts at 2024-09-10 19:13:09.280595 (724329.404575905)
+Experiment                    : This experiment compares run time and resource consumption of Benchbase queries in different DBMS.
+Benchbase data is generated and loaded using several threads.
+Benchmark is 'tpcc'. Scaling factor (e.g., number of warehouses) is 16. Benchmarking runs for 300 minutes.
+Benchmark is limited to DBMS PostgreSQL.
+Loading is fixed to cl-worker19.
+Benchmarking is fixed to cl-worker19. SUT is fixed to cl-worker11.
+Loading is tested with [1] threads and [1] target factors of base 1024, split into [1] pods.
+Benchmarking is tested with [16] threads and [8] target factors of base 1024, split into [1] pods.
+Benchmarking is run as [1] times the number of benchmarking pods.
+Experiment is run once.
+Cluster monitoring            : is running
+done
+PostgreSQL-1-1-1024           : will start now
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is not loaded yet
+PostgreSQL-1-1-1024           : will start loading but not before 2024-09-10 17:15:09 (that is in 60 secs)
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is not loaded yet
+PostgreSQL-1-1-1024           : will start loading but not before 2024-09-10 17:15:09
+done
+PostgreSQL-1-1-1024           : is not loaded yet
+PostgreSQL-1-1-1024           : start asynch loading scripts of type loaded
+PostgreSQL-1-1-1024           : is loading
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is loading
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is loading
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is loading
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is loading
+done
+PostgreSQL-1-1-1024           : showing loader times
+PostgreSQL-1-1-1024           : loader timespan (first to last [s]) = 134
+PostgreSQL-1-1-1024           : benchmarks done 0 of 1. This will be client 1
+PostgreSQL-1-1-1024           : we will change parameters of benchmark as {'PARALLEL': '1', 'SF': '16', 'BENCHBASE_BENCH': 'tpcc', 'BENCHBASE_PROFILE': 'postgres', 'BEXHOMA_DATABASE': 'postgres', 'BENCHBASE_TARGET': 8192, 'BENCHBASE_TERMINALS': 16, 'BENCHBASE_TIME': 300, 'BENCHBASE_ISOLATION': 'TRANSACTION_READ_COMMITTED'}
+PostgreSQL-1-1-1024-1         : start benchmarking
+PostgreSQL-1-1-1024           : benchmarking results in folder /home/perdelt/benchmarks/1725988389
+- waiting 10s -               : done
+PostgreSQL-1-1-1024           : benchmarking is waiting for job bexhoma-benchmarker-postgresql-1-1-1024-1725988389-1-1-z9dwb: found
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+done
+PostgreSQL-1-1-1024           : has running benchmarks
+PostgreSQL-1-1-1024-1         : showing benchmarker times
+PostgreSQL-1-1-1024-1         : benchmarker timespan (start to end single container [s]) = 342
+PostgreSQL-1-1-1024-1         : benchmarker times (start/end per pod and container) = [(1725988666, 1725988977)]
+PostgreSQL-1-1-1024-1         : found and updated times {'benchmarker': [(1725988666, 1725988977)]}
+/home/perdelt/repositories/Benchmark-Experiment-Host-Manager/bexhoma/experiments.py:2276: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
+  df.fillna(0, inplace=True)
+done
+PostgreSQL-1-1-1024           : can be stopped
+Experiment ends at 2024-09-10 19:23:57.350672 (724977.474652074): 648.0700761690969s total
+bexhoma-dashboard-9b85f945f-6nwk6 Running
+Benchmarking                                     connection  ... Latency Distribution.Average Latency (microseconds)
+connection_pod                                  ...                                                    
+PostgreSQL-1-1-1024-1-1  PostgreSQL-1-1-1024-1  ...                                                6076
+
+[1 rows x 36 columns]
+Workflow {'PostgreSQL-1-1-1024': [[1]]}
+Result workflow complete
+
+## Show Summary
+
+### Workload
+    Benchbase Workload SF=16 (warehouses for TPC-C)
+    This includes no queries. Benchbase runs the benchmark
+    This experiment compares run time and resource consumption of Benchbase queries in different DBMS.
+Benchbase data is generated and loaded using several threads.
+Benchmark is 'tpcc'. Scaling factor (e.g., number of warehouses) is 16. Benchmarking runs for 300 minutes.
+Benchmark is limited to DBMS PostgreSQL.
+Loading is fixed to cl-worker19.
+Benchmarking is fixed to cl-worker19. SUT is fixed to cl-worker11.
+Loading is tested with [1] threads and [1] target factors of base 1024, split into [1] pods.
+Benchmarking is tested with [16] threads and [8] target factors of base 1024, split into [1] pods.
+Benchmarking is run as [1] times the number of benchmarking pods.
+Experiment is run once.
+
+### Connections
+PostgreSQL-1-1-1024-1 uses docker image postgres:16.1
+    RAM:541008605184
+    CPU:AMD Opteron(tm) Processor 6378
+    Cores:64
+    host:5.15.0-116-generic
+    node:cl-worker11
+    disk:250758688
+    datadisk:4409124
+    requests_cpu:4
+    requests_memory:16Gi
+
+### Execution
+                       experiment_run  terminals  target  pod_count   time  Throughput (requests/second)  Latency Distribution.95th Percentile Latency (microseconds)  Latency Distribution.Average Latency (microseconds)
+PostgreSQL-1-1-1024-1               1         16    8192          1  300.0                       2630.71                                                      13258.0                                               6076.0
+
+Warehouses: 16
+
+### Workflow
+DBMS PostgreSQL-1-1-1024 - Pods [[1]]
+
+### Loading
+                       time_load  terminals  pods  Imported warehouses [1/h]
+PostgreSQL-1-1-1024-1      134.0        1.0   1.0                 429.850746
+TEST passed: Throughput (requests/second) contains no 0 or NaN
diff --git a/logs_tests/test_benchbase_testcase_1_summary.txt b/logs_tests/test_benchbase_testcase_1_summary.txt
@@ -0,0 +1,41 @@
+## Show Summary
+
+### Workload
+    Benchbase Workload SF=16 (warehouses for TPC-C)
+    This includes no queries. Benchbase runs the benchmark
+    This experiment compares run time and resource consumption of Benchbase queries in different DBMS.
+Benchbase data is generated and loaded using several threads.
+Benchmark is 'tpcc'. Scaling factor (e.g., number of warehouses) is 16. Benchmarking runs for 300 minutes.
+Benchmark is limited to DBMS PostgreSQL.
+Loading is fixed to cl-worker19.
+Benchmarking is fixed to cl-worker19. SUT is fixed to cl-worker11.
+Loading is tested with [1] threads and [1] target factors of base 1024, split into [1] pods.
+Benchmarking is tested with [16] threads and [8] target factors of base 1024, split into [1] pods.
+Benchmarking is run as [1] times the number of benchmarking pods.
+Experiment is run once.
+
+### Connections
+PostgreSQL-1-1-1024-1 uses docker image postgres:16.1
+    RAM:541008605184
+    CPU:AMD Opteron(tm) Processor 6378
+    Cores:64
+    host:5.15.0-116-generic
+    node:cl-worker11
+    disk:250758688
+    datadisk:4409124
+    requests_cpu:4
+    requests_memory:16Gi
+
+### Execution
+                       experiment_run  terminals  target  pod_count   time  Throughput (requests/second)  Latency Distribution.95th Percentile Latency (microseconds)  Latency Distribution.Average Latency (microseconds)
+PostgreSQL-1-1-1024-1               1         16    8192          1  300.0                       2630.71                                                      13258.0                                               6076.0
+
+Warehouses: 16
+
+### Workflow
+DBMS PostgreSQL-1-1-1024 - Pods [[1]]
+
+### Loading
+                       time_load  terminals  pods  Imported warehouses [1/h]
+PostgreSQL-1-1-1024-1      134.0        1.0   1.0                 429.850746
+TEST passed: Throughput (requests/second) contains no 0 or NaN
diff --git a/logs_tests/test_benchbase_testcase_2.log b/logs_tests/test_benchbase_testcase_2.log
@@ -0,0 +1,150 @@
+Dashboard                     : is running
+Experiment                    : has code 1725989299
+Experiment                    : starts at 2024-09-10 19:28:19.366032 (725239.490013773)
+Experiment                    : This experiment compares run time and resource consumption of Benchbase queries in different DBMS.
+Benchbase data is generated and loaded using several threads.
+Benchmark is 'tpcc'. Scaling factor (e.g., number of warehouses) is 16. Benchmarking runs for 60 minutes.
+Benchmark is limited to DBMS PostgreSQL.
+Loading is fixed to cl-worker19.
+Benchmarking is fixed to cl-worker19. SUT is fixed to cl-worker11.
+Database is persisted to disk of type shared and size 50Gi.
+Loading is tested with [1] threads and [1] target factors of base 1024, split into [1] pods.
+Benchmarking is tested with [16] threads and [8] target factors of base 1024, split into [1] pods.
+Benchmarking is run as [1] times the number of benchmarking pods.
+Experiment is run 2 times.
+Cluster monitoring            : is running
+done
+PostgreSQL-1-1-1024           : will start now
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is not loaded yet
+PostgreSQL-1-1-1024           : will start loading but not before 2024-09-10 17:30:19 (that is in 60 secs)
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is not loaded yet
+PostgreSQL-1-1-1024           : will start loading but not before 2024-09-10 17:30:19
+done
+PostgreSQL-1-1-1024           : is not loaded yet
+PostgreSQL-1-1-1024           : start asynch loading scripts of type loaded
+PostgreSQL-1-1-1024           : is loading
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is loading
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is loading
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is loading
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : is loading
+done
+PostgreSQL-1-1-1024           : showing loader times
+PostgreSQL-1-1-1024           : loader timespan (first to last [s]) = 134
+PostgreSQL-1-1-1024           : benchmarks done 0 of 2. This will be client 1
+PostgreSQL-1-1-1024           : we will change parameters of benchmark as {'PARALLEL': '1', 'SF': '16', 'BENCHBASE_BENCH': 'tpcc', 'BENCHBASE_PROFILE': 'postgres', 'BEXHOMA_DATABASE': 'postgres', 'BENCHBASE_TARGET': 8192, 'BENCHBASE_TERMINALS': 16, 'BENCHBASE_TIME': 60, 'BENCHBASE_ISOLATION': 'TRANSACTION_READ_COMMITTED'}
+PostgreSQL-1-1-1024-1-1       : start benchmarking
+PostgreSQL-1-1-1024           : benchmarking results in folder /home/perdelt/benchmarks/1725989299
+- waiting 10s -               : done
+PostgreSQL-1-1-1024           : benchmarking is waiting for job bexhoma-benchmarker-postgresql-1-1-1024-1725989299-1-1-vjxm5: found
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+done
+PostgreSQL-1-1-1024           : has running benchmarks
+PostgreSQL-1-1-1024-1-1       : showing benchmarker times
+PostgreSQL-1-1-1024-1-1       : benchmarker timespan (start to end single container [s]) = 71
+PostgreSQL-1-1-1024-1-1       : benchmarker times (start/end per pod and container) = [(1725989576, 1725989639)]
+PostgreSQL-1-1-1024-1-1       : found and updated times {'benchmarker': [(1725989576, 1725989639)]}
+done
+PostgreSQL-1-1-1024           : can be stopped
+PostgreSQL-1-1-1024           : still being removed
+done
+PostgreSQL-1-1-1024           : starts again
+PostgreSQL-1-1-1024           : storage exists bexhoma-storage-postgresql-benchbase-16
+PostgreSQL-1-1-1024           : loading is set to finished
+- waiting 10s -               : done
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : will start benchmarking but not before 2024-09-10 17:36:46 (that is in 60 secs)
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : will start benchmarking but not before 2024-09-10 17:36:46
+done
+PostgreSQL-1-1-1024           : benchmarks done 1 of 2. This will be client 1
+PostgreSQL-1-1-1024           : we will change parameters of benchmark as {'PARALLEL': '1', 'SF': '16', 'BENCHBASE_BENCH': 'tpcc', 'BENCHBASE_PROFILE': 'postgres', 'BEXHOMA_DATABASE': 'postgres', 'BENCHBASE_TARGET': 8192, 'BENCHBASE_TERMINALS': 16, 'BENCHBASE_TIME': 60, 'BENCHBASE_ISOLATION': 'TRANSACTION_READ_COMMITTED'}
+PostgreSQL-1-1-1024-2-1       : start benchmarking
+PostgreSQL-1-1-1024           : benchmarking results in folder /home/perdelt/benchmarks/1725989299
+- waiting 10s -               : done
+PostgreSQL-1-1-1024           : benchmarking is waiting for job bexhoma-benchmarker-postgresql-1-1-1024-1725989299-2-1-8mqmc: found
+- waiting 30s -               : done
+PostgreSQL-1-1-1024           : has running benchmarks
+done
+PostgreSQL-1-1-1024           : has running benchmarks
+PostgreSQL-1-1-1024-2-1       : showing benchmarker times
+PostgreSQL-1-1-1024-2-1       : benchmarker timespan (start to end single container [s]) = 70
+PostgreSQL-1-1-1024-2-1       : benchmarker times (start/end per pod and container) = [(1725989811, 1725989875)]
+PostgreSQL-1-1-1024-2-1       : found and updated times {'benchmarker': [(1725989811, 1725989875)]}
+done
+PostgreSQL-1-1-1024           : can be stopped
+Experiment ends at 2024-09-10 19:38:31.621876 (725851.745856264): 612.255842491053s total
+bexhoma-dashboard-9b85f945f-g5pzh Running
+Benchmarking                                         connection  ... Latency Distribution.Average Latency (microseconds)
+connection_pod                                      ...                                                    
+PostgreSQL-1-1-1024-1-1-1  PostgreSQL-1-1-1024-1-1  ...                                                6400
+PostgreSQL-1-1-1024-2-1-1  PostgreSQL-1-1-1024-2-1  ...                                               11446
+
+[2 rows x 36 columns]
+Workflow {'PostgreSQL-1-1-1024': [[1], [1]]}
+Result workflow complete
+
+## Show Summary
+
+### Workload
+    Benchbase Workload SF=16 (warehouses for TPC-C)
+    This includes no queries. Benchbase runs the benchmark
+    This experiment compares run time and resource consumption of Benchbase queries in different DBMS.
+Benchbase data is generated and loaded using several threads.
+Benchmark is 'tpcc'. Scaling factor (e.g., number of warehouses) is 16. Benchmarking runs for 60 minutes.
+Benchmark is limited to DBMS PostgreSQL.
+Loading is fixed to cl-worker19.
+Benchmarking is fixed to cl-worker19. SUT is fixed to cl-worker11.
+Database is persisted to disk of type shared and size 50Gi.
+Loading is tested with [1] threads and [1] target factors of base 1024, split into [1] pods.
+Benchmarking is tested with [16] threads and [8] target factors of base 1024, split into [1] pods.
+Benchmarking is run as [1] times the number of benchmarking pods.
+Experiment is run 2 times.
+
+### Connections
+PostgreSQL-1-1-1024-1-1 uses docker image postgres:16.1
+    RAM:541008605184
+    CPU:AMD Opteron(tm) Processor 6378
+    Cores:64
+    host:5.15.0-116-generic
+    node:cl-worker11
+    disk:246349560
+    datadisk:4409032
+    volume_size:50G
+    volume_used:4.2G
+    requests_cpu:4
+    requests_memory:16Gi
+PostgreSQL-1-1-1024-2-1 uses docker image postgres:16.1
+    RAM:541008605184
+    CPU:AMD Opteron(tm) Processor 6378
+    Cores:64
+    host:5.15.0-116-generic
+    node:cl-worker11
+    disk:246349560
+    datadisk:5108384
+    volume_size:50G
+    volume_used:4.9G
+    requests_cpu:4
+    requests_memory:16Gi
+
+### Execution
+                         experiment_run  terminals  target  pod_count  time  Throughput (requests/second)  Latency Distribution.95th Percentile Latency (microseconds)  Latency Distribution.Average Latency (microseconds)
+PostgreSQL-1-1-1024-1-1               1         16    8192          1  60.0                       2495.83                                                      13974.0                                               6400.0
+PostgreSQL-1-1-1024-2-1               2         16    8192          1  60.0                       1396.38                                                      14976.0                                              11446.0
+
+Warehouses: 16
+
+### Workflow
+DBMS PostgreSQL-1-1-1024 - Pods [[1], [1]]
+
+### Loading
+                         time_load  terminals  pods  Imported warehouses [1/h]
+PostgreSQL-1-1-1024-1-1      134.0        1.0   1.0                 429.850746
+PostgreSQL-1-1-1024-2-1      134.0        1.0   1.0                 429.850746
+TEST passed: Throughput (requests/second) contains no 0 or NaN