Merge branch 'v0.7.1' into snyk-fix-423a522fe215f3155b4794460286bd98

.github/workflows/publish-to-test-pypi.yml

@@ -4,8 +4,9 @@ name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
 
 # Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the master branch
-  push:
+  # Triggers the workflow on release events but only for the master branch
+  release:
+    types: [published]
     branches: [ master ]
     tags:
       - "v*"
@@ -15,8 +16,8 @@ on:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
-  build:
+  # This workflow contains a single job called "release"
+  release:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
 

.gitignore

@@ -11,3 +11,5 @@ dist/*
 .spyproject/*
 docs/html/*
 api/*
+bexhoma/__pycache__/*
+/cluster-monitoring-default.config

README.md

@@ -6,6 +6,9 @@
 [![Documentation Status](https://readthedocs.org/projects/bexhoma/badge/?version=latest)](https://bexhoma.readthedocs.io/en/latest/?badge=latest)
 
 # Benchmark Experiment Host Manager (Bexhoma)
+
+## Orchestrating Cloud-Native Benchmarking Experiments with Kubernetes
+
 This Python tools helps **managing benchmark experiments of Database Management Systems (DBMS) in a Kubernetes-based High-Performance-Computing (HPC) cluster environment**.
 It enables users to configure hardware / software setups for easily repeating tests over varying configurations.
 
@@ -23,7 +26,7 @@ The basic workflow is [1,2]: start a containerized version of the DBMS, install
 A more advanced workflow is: Plan a sequence of such experiments, run plan as a batch and join results for comparison.
 
 It is also possible to scale-out drivers for generating and loading data and for benchmarking to simulate cloud-native environments as in [4].
-See [example](https://github.com/Beuth-Erdelt/Benchmark-Experiment-Host-Manager/TPCTC23/README.md) results as presented in [A Cloud-Native Adoption of Classical DBMS Performance Benchmarks and Tools](http://dx.doi.org/10.13140/RG.2.2.29866.18880) and how they are generated.
+See [example](https://github.com/Beuth-Erdelt/Benchmark-Experiment-Host-Manager/tree/master/TPCTC23) results as presented in [A Cloud-Native Adoption of Classical DBMS Performance Benchmarks and Tools](http://dx.doi.org/10.13140/RG.2.2.29866.18880) and how they are generated.
 
 See the [homepage](https://github.com/Beuth-Erdelt/Benchmark-Experiment-Host-Manager) and the [documentation](https://bexhoma.readthedocs.io/en/latest/).
 
@@ -55,19 +58,19 @@ If you encounter any issues, please report them to our [Github issue tracker](ht
 
 1. Run `python ycsb.py -ms 1 -dbms PostgreSQL -workload a run`.  
   This installs PostgreSQL and runs YCSB workload A with varying target. The driver is monolithic with 64 threads. The experiments runs a second time with the driver scaled out to 8 instances each having 8 threads.
-1. You can watch status using `bexperiments status` while running. This is equivalent to `python cluster.py status`.
+1. You can watch status using `bexperiments status` while running.
 1. After benchmarking has finished, you will see a summary.  
-  For further inspections, run `bexperiments dashboard` to connect to a dashboard. You can open dashboard in browser at `http://localhost:8050`. This is equivalent to `python cluster.py dashboard`. Alternatively you can open a Jupyter notebook at `http://localhost:8888`.
+  For further inspections, run `bexperiments dashboard` to connect to a dashboard. You can open dashboard in browser at `http://localhost:8050`. Alternatively you can open a Jupyter notebook at `http://localhost:8888`.
 
 See more details at https://bexhoma.readthedocs.io/en/latest/Example-YCSB.html
 
 ### TPC-H
 
 1. Run `python tpch.py -ms 1 -dbms PostgreSQL run`.  
   This installs PostgreSQL and runs TPC-H at scale factor 1. The driver is monolithic.
-1. You can watch status using `bexperiments status` while running. This is equivalent to `python cluster.py status`.
+1. You can watch status using `bexperiments status` while running.
 1. After benchmarking has finished, you will see a summary.  
-  For further inspections, run `bexperiments dashboard` to connect to a dashboard. This is equivalent to `python cluster.py dashboard`.  You can open a Jupyter notebook at `http://localhost:8888`.
+  For further inspections, run `bexperiments dashboard` to connect to a dashboard. You can open a Jupyter notebook at `http://localhost:8888`.
 
 See more details at https://bexhoma.readthedocs.io/en/latest/Example-TPC-H.html
 

TPCTC23/experiment-1-ycsb.py

@@ -5,7 +5,6 @@
 """
 from bexhoma import *
 from dbmsbenchmarker import *
-#import experiments
 import logging
 import urllib3
 import logging

TPCTC23/experiment-2-hammerdb.py

@@ -9,7 +9,6 @@
 """
 from bexhoma import *
 from dbmsbenchmarker import *
-#import experiments
 import logging
 import urllib3
 import logging

TPCTC23/experiment-3-benchbase.py

@@ -9,7 +9,6 @@
 """
 from bexhoma import *
 from dbmsbenchmarker import *
-#import experiments
 import logging
 import urllib3
 import logging

TPCTC23/experiment-4-1-tpch-generating-filesystem.py

@@ -34,12 +34,10 @@
 import time
 from timeit import default_timer
 import datetime
-# queue
-import redis
 import subprocess
 import psutil
 
-urllib3.disable_warnings()
+#urllib3.disable_warnings()
 logging.basicConfig(level=logging.ERROR)
 
 if __name__ == '__main__':

TPCTC23/experiment-4-2-tpch-loading-filesystem.py

@@ -40,8 +40,6 @@
 import time
 from timeit import default_timer
 import datetime
-# queue
-import redis
 import subprocess
 import psutil
 

TPCTC23/experiment-4-3-tpch-benchmarking.py

@@ -40,8 +40,6 @@
 import time
 from timeit import default_timer
 import datetime
-# queue
-import redis
 import subprocess
 import psutil
 

bexhoma/clusters.py

@@ -43,6 +43,8 @@
 import json
 import ast
 import copy
+import urllib.request
+import urllib.parse
 
 from dbmsbenchmarker import *
 from bexhoma import experiments
@@ -107,6 +109,7 @@ def __init__(self, clusterconfig='cluster.config', experiments_configfolder='exp
         self.port = self.contextdata['port']
         self.monitoring_active = True
         self.monitor_cluster_active = False
+        self.monitor_cluster_exists = False                                                     # True, if there are cAdvisors and a Prometheus server independent from bexhoma
         # k8s:
         self.namespace = self.contextdata['namespace']
         self.appname = self.config['credentials']['k8s']['appname']
@@ -846,7 +849,9 @@ def execute_command_in_pod(self, command, pod='', container='', params=''):
         :return: stdout of the shell command
         """
         if len(pod) == 0:
-            pod = self.activepod
+            self.logger.debug('testbed.execute_command_in_pod({}): empty pod name given for command'.format(command))
+            return "", "", ""
+            #pod = self.activepod
         command_clean = command.replace('"','\\"')
         if len(container) > 0:
             fullcommand = 'kubectl --context {context} exec {pod} --container={container} -- bash -c "{command}"'.format(context=self.context, pod=pod, container=container, command=command_clean)
@@ -966,7 +971,7 @@ def copyInits(self):
         scriptfolder = '/data/{experiment}/{docker}/'.format(experiment=self.experiments_configfolder, docker=self.d)
         i = 0
         for script in self.initscript:
-            cmd['copy_init_scripts'] = 'cp {scriptname}'.format(scriptname=scriptfolder+script)+' /data/'+str(self.code)+'/'+self.connection+'_init_'+str(i)+'.log'
+            cmd['copy_init_scripts'] = 'cp {scriptname}'.format(scriptname=scriptfolder+script, namespace=self.namespace)+' /data/'+str(self.code)+'/'+self.connection+'_init_'+str(i)+'.log'
             stdin, stdout, stderr = self.execute_command_in_pod(cmd['copy_init_scripts'], container='dbms')
             i = i + 1
     def pod_log(self, pod, container=''):
@@ -1308,6 +1313,56 @@ def start_dashboard(self, app='', component='dashboard'):
                self.wait(10, silent=True)
             print("done")
             return
+    def test_if_monitoring_healthy(self):
+        """
+        Tests if query_range?query=node_memory_MemTotal_bytes&start=1&end=2&step=1 at service_monitoring returns status code of 200.
+        This is for testing if Prometheus is up and running.
+
+        :return: True if Prometheus returns status code 200
+        """
+        self.logger.debug('testbed.test_if_monitoring_healthy()')
+        config_K8s = self.config['credentials']['k8s']
+        if 'service_monitoring' in config_K8s['monitor']:
+            url = config_K8s['monitor']['service_monitoring'].format(namespace=self.contextdata['namespace'], service="monitoring")
+            query = "node_memory_MemTotal_bytes"
+            safe_query = urllib.parse.quote_plus(query)
+            try:
+                self.logger.debug('Test URL {}'.format(url+"query_range?query="+safe_query+"&start=1&end=2&step=1"))
+                #code= urllib.request.urlopen(url+"query_range?query="+safe_query+"&start=1&end=2&step=1").getcode()
+                # curl -ILs www.welt.de | head -n 1|cut -d$' ' -f2
+                pod_dashboard = self.get_dashboard_pod_name()
+                self.logger.debug('Inside pod {}'.format(pod_dashboard))
+                cmd = {}
+                command = "curl -is '{}' | head -n 1|cut -d$' ' -f2".format(url+"query_range?query="+safe_query+"&start=1&end=2&step=1")
+                self.logger.debug('Command {}'.format(command))
+                #fullcommand = 'kubectl exec '+self.pod_sut+' --container=dbms -- bash -c "'+command+'"'
+                #cores = os.popen(fullcommand).read()
+                stdin, stdout, stderr = self.execute_command_in_pod(pod=pod_dashboard, command=command, container="dashboard")
+                #print("Return", stdout, stderr)
+                status = stdout#os.popen(fullcommand).read()
+                if len(status)>0:
+                    #return int(status)
+                    #print(int(status))
+                    if int(status) == 200:
+                        return True
+                    else:
+                        return False
+                else:
+                    return False
+                #except Exception as e:
+                #    logging.error(e)
+                #    return 0
+                # curl -I http://www.example.org
+                #if code == 200:
+                #    #print("{:30s}: is running".format("Prometheus"))
+                #    return True
+                #else:
+                #    #print("{:30s}: is not running".format("Prometheus"))
+                #    return False
+            except Exception as e:
+                #print("{:30s}: is not running".format("Prometheus"))
+                print(e)
+                return False
     def start_monitoring_cluster(self, app='', component='monitoring'):
         """
         Starts the monitoring component and its service.
@@ -1317,6 +1372,9 @@ def start_monitoring_cluster(self, app='', component='monitoring'):
         :param component: Component name, should be 'monitoring' typically
         """
         self.monitor_cluster_active = True
+        self.monitor_cluster_exists = self.test_if_monitoring_healthy()
+        if self.monitor_cluster_exists:
+            return
         endpoints = self.get_service_endpoints(service_name="bexhoma-service-monitoring-default")
         if len(endpoints) > 0:
             # monitoring exists