Merge pull request #726 from deNBI/staging

Staging

.github/workflows/main.yml

@@ -2,7 +2,9 @@ name: Publish Docker
 on: 
   push:
     branches: 
+      - 'master'
       - 'staging'
+      - 'dev'
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -12,11 +14,16 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - uses: actions/checkout@master
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+
     - name: Publish to Registry
       uses: elgohr/Publish-Docker-Github-Action@master
       with:
-        name: denbicloud/cloud-portal-client
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-        dockerfile: Dockerfile
-        tags: "staging"
+              name: denbicloud/cloud-portal-client
+              username: ${{ secrets.DOCKER_USERNAME }}
+              password: ${{ secrets.DOCKER_PASSWORD }}
+              dockerfile: Dockerfile
+              tags: ${{ steps.extract_branch.outputs.branch }}

Dockerfile.update

@@ -1,3 +1,3 @@
 FROM denbicloud/cloud-portal-client:0.1.0-beta.0.15.1
 FROM bibiserv/bibigrid:bibigrid-rest-2.3
-FROM docker.elastic.co/beats/filebeat:7.12.1
+FROM docker.elastic.co/beats/filebeat:7.13.0

VirtualMachineService/VirtualMachineHandler.py

@@ -3,6 +3,7 @@
 
 Which can be used for the PortalClient.
 """
+import sys
 from uuid import uuid4
 
 try:
@@ -151,10 +152,6 @@ def __init__(self, config):
         Read all config variables and creates a connection to OpenStack.
         """
 
-        # connection to redis. Uses a pool with 10 connections.
-        self.pool = redis.ConnectionPool(host="redis", port=6379)
-        self.redis = redis.Redis(connection_pool=self.pool, charset="utf-8")
-
         self.USERNAME = os.environ["OS_USERNAME"]
         self.PASSWORD = os.environ["OS_PASSWORD"]
         self.PROJECT_NAME = os.environ["OS_PROJECT_NAME"]
@@ -174,6 +171,23 @@ def __init__(self, config):
             self.AVAIALABILITY_ZONE = cfg["openstack_connection"]["availability_zone"]
             self.PRODUCTION = cfg["openstack_connection"]["production"]
             self.CLOUD_SITE = cfg["cloud_site"]
+            # connection to redis. Uses a pool with 10 connections.
+            self.REDIS_HOST = cfg["redis"]["host"]
+            self.REDIS_PORT = cfg["redis"]["port"]
+            self.REDIS_PASSWORD = cfg["redis"]["password"]
+            LOG.info(f"Connecting to Redis at {self.REDIS_HOST}:{self.REDIS_PORT}..")
+            self.pool = redis.ConnectionPool(
+                host=self.REDIS_HOST, port=self.REDIS_PORT, password=self.REDIS_PASSWORD
+            )
+
+            self.redis = redis.Redis(connection_pool=self.pool, charset="utf-8")
+            try:
+                self.redis.ping()
+                LOG.info("Connected to Redis!")
+            except redis.ConnectionError:
+                LOG.exception("Could not connect to Redis!")
+                sys.exit(1)
+
             # try to initialize forc connection
             try:
                 self.SUB_NETWORK = cfg["bibigrid"]["sub_network"]

VirtualMachineService/VirtualMachineServer.py

@@ -12,7 +12,7 @@
     print(e)
     from VirtualMachineHandler import VirtualMachineHandler
 
-from thrift.transport import TSSLSocket
+from thrift.transport import TSSLSocket, TSocket
 from thrift.transport import TTransport
 from thrift.protocol import TBinaryProtocol
 from thrift.server import TServer
@@ -62,10 +62,16 @@ def catch_shutdown(signal, frame):
         PORT = cfg["openstack_connection"]["port"]
         CERTFILE = cfg["openstack_connection"]["certfile"]
         THREADS = cfg["openstack_connection"]["threads"]
+        USE_SSL = cfg["openstack_connection"].get("use_ssl", True)
     click.echo("Server is running on port {}".format(PORT))
     handler = VirtualMachineHandler(CONFIG_FILE)
     processor = Processor(handler)
-    transport = TSSLSocket.TSSLServerSocket(host=HOST, port=PORT, certfile=CERTFILE)
+    if USE_SSL:
+        click.echo("Use SSL")
+        transport = TSSLSocket.TSSLServerSocket(host=HOST, port=PORT, certfile=CERTFILE)
+    else:
+        click.echo("Does not use SSL")
+        transport = TSocket.TServerSocket(host=HOST, port=PORT)
     tfactory = TTransport.TBufferedTransportFactory()
     pfactory = TBinaryProtocol.TBinaryProtocolFactory()
     server = TServer.TThreadPoolServer(

VirtualMachineService/ancon/Playbook.py

@@ -228,9 +228,12 @@ def add_always_tasks_only(self, playbook_name):
         )
 
     def run_it(self):
-        command_string = "/usr/local/bin/ansible-playbook -v -i {0} {1}/{2}".format(
-            self.inventory.name, self.directory.name, self.playbook_exec_name
+        command_string = (
+            "/usr/local/bin/ansible-playbook -t 30 -vvvv -i {0} {1}/{2}".format(
+                self.inventory.name, self.directory.name, self.playbook_exec_name
+            )
         )
+        LOG.info(f"Run Playbook with command {command_string}")
         command_string = shlex.split(command_string)
         self.process = subprocess.Popen(
             command_string,

VirtualMachineService/config/config.yml

@@ -1,3 +1,8 @@
+redis:
+  host: redis
+  port: 6379
+  password: ""
+
 openstack_connection:
   threads: 30
   host: 0.0.0.0
@@ -24,6 +29,7 @@ openstack_connection:
   availability_zone: default
   # If you use docker-compose this path needs to be the path you mount the server.pem into
   certfile: /code/VirtualMachineService/keys/server.pem
+  use_ssl: True
 
   # Used for gateway port calculation , x is always the last octet of the fixed ip (example : (x + 3) *3 ) )
   ssh_port_calc_formular: 30000 + x
@@ -43,4 +49,4 @@ forc:
   forc_api_key:
   github_playbooks_repo:
 
-cloud_site: bielefeld
+cloud_site: bielefeld

ansible.cfg

@@ -1,8 +1,10 @@
 [defaults]
 host_key_checking = False
+record_host_key = False
 roles_path = ~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:/code/VirtualMachineService/ancon/playbooks/roles
 
 [paramiko_connection]
 host_key_checking = False
+record_host_key = False
 env:ANSIBLE_PARAMIKO_HOST_KEY_CHECKING
 var: ansible_paramiko_host_key_checking

requirements.txt

@@ -1,10 +1,10 @@
-setuptools==56.2.0
+setuptools==57.0.0
 thrift >= 0.11.0,<0.20.0
 python-keystoneclient
-openstacksdk ==0.56.0
+openstacksdk ==0.57.0
 deprecated == 1.2.12
-Click==7.1.2
-ansible==3.4.0
+Click==8.0.1
+ansible==4.0.0
 flake8==3.9.2
 ruamel.yaml<0.18.0
 paramiko==2.7.2