diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b67f12f0..2aea28ff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,7 +2,9 @@ name: Publish Docker on: push: branches: + - 'master' - 'staging' + - 'dev' jobs: build: runs-on: ubuntu-latest @@ -12,11 +14,16 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: actions/checkout@master + - name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + - name: Publish to Registry uses: elgohr/Publish-Docker-Github-Action@master with: - name: denbicloud/cloud-portal-client - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - dockerfile: Dockerfile - tags: "staging" + name: denbicloud/cloud-portal-client + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + dockerfile: Dockerfile + tags: ${{ steps.extract_branch.outputs.branch }} diff --git a/Dockerfile.update b/Dockerfile.update index 74a8f2a0..4a309cab 100644 --- a/Dockerfile.update +++ b/Dockerfile.update @@ -1,3 +1,3 @@ FROM denbicloud/cloud-portal-client:0.1.0-beta.0.15.1 FROM bibiserv/bibigrid:bibigrid-rest-2.3 -FROM docker.elastic.co/beats/filebeat:7.12.1 +FROM docker.elastic.co/beats/filebeat:7.13.0 diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 3c6be81e..d86c15d1 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -3,6 +3,7 @@ Which can be used for the PortalClient. """ +import sys from uuid import uuid4 try: @@ -151,10 +152,6 @@ def __init__(self, config): Read all config variables and creates a connection to OpenStack. """ - # connection to redis. Uses a pool with 10 connections. - self.pool = redis.ConnectionPool(host="redis", port=6379) - self.redis = redis.Redis(connection_pool=self.pool, charset="utf-8") - self.USERNAME = os.environ["OS_USERNAME"] self.PASSWORD = os.environ["OS_PASSWORD"] self.PROJECT_NAME = os.environ["OS_PROJECT_NAME"] @@ -174,6 +171,23 @@ def __init__(self, config): self.AVAIALABILITY_ZONE = cfg["openstack_connection"]["availability_zone"] self.PRODUCTION = cfg["openstack_connection"]["production"] self.CLOUD_SITE = cfg["cloud_site"] + # connection to redis. Uses a pool with 10 connections. + self.REDIS_HOST = cfg["redis"]["host"] + self.REDIS_PORT = cfg["redis"]["port"] + self.REDIS_PASSWORD = cfg["redis"]["password"] + LOG.info(f"Connecting to Redis at {self.REDIS_HOST}:{self.REDIS_PORT}..") + self.pool = redis.ConnectionPool( + host=self.REDIS_HOST, port=self.REDIS_PORT, password=self.REDIS_PASSWORD + ) + + self.redis = redis.Redis(connection_pool=self.pool, charset="utf-8") + try: + self.redis.ping() + LOG.info("Connected to Redis!") + except redis.ConnectionError: + LOG.exception("Could not connect to Redis!") + sys.exit(1) + # try to initialize forc connection try: self.SUB_NETWORK = cfg["bibigrid"]["sub_network"] diff --git a/VirtualMachineService/VirtualMachineServer.py b/VirtualMachineService/VirtualMachineServer.py index 00d0d916..1eb9020d 100644 --- a/VirtualMachineService/VirtualMachineServer.py +++ b/VirtualMachineService/VirtualMachineServer.py @@ -12,7 +12,7 @@ print(e) from VirtualMachineHandler import VirtualMachineHandler -from thrift.transport import TSSLSocket +from thrift.transport import TSSLSocket, TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol from thrift.server import TServer @@ -62,10 +62,16 @@ def catch_shutdown(signal, frame): PORT = cfg["openstack_connection"]["port"] CERTFILE = cfg["openstack_connection"]["certfile"] THREADS = cfg["openstack_connection"]["threads"] + USE_SSL = cfg["openstack_connection"].get("use_ssl", True) click.echo("Server is running on port {}".format(PORT)) handler = VirtualMachineHandler(CONFIG_FILE) processor = Processor(handler) - transport = TSSLSocket.TSSLServerSocket(host=HOST, port=PORT, certfile=CERTFILE) + if USE_SSL: + click.echo("Use SSL") + transport = TSSLSocket.TSSLServerSocket(host=HOST, port=PORT, certfile=CERTFILE) + else: + click.echo("Does not use SSL") + transport = TSocket.TServerSocket(host=HOST, port=PORT) tfactory = TTransport.TBufferedTransportFactory() pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TThreadPoolServer( diff --git a/VirtualMachineService/ancon/Playbook.py b/VirtualMachineService/ancon/Playbook.py index 6f902d1f..8741ca6d 100644 --- a/VirtualMachineService/ancon/Playbook.py +++ b/VirtualMachineService/ancon/Playbook.py @@ -228,9 +228,12 @@ def add_always_tasks_only(self, playbook_name): ) def run_it(self): - command_string = "/usr/local/bin/ansible-playbook -v -i {0} {1}/{2}".format( - self.inventory.name, self.directory.name, self.playbook_exec_name + command_string = ( + "/usr/local/bin/ansible-playbook -t 30 -vvvv -i {0} {1}/{2}".format( + self.inventory.name, self.directory.name, self.playbook_exec_name + ) ) + LOG.info(f"Run Playbook with command {command_string}") command_string = shlex.split(command_string) self.process = subprocess.Popen( command_string, diff --git a/VirtualMachineService/config/config.yml b/VirtualMachineService/config/config.yml index d5e2213b..8f43eb19 100644 --- a/VirtualMachineService/config/config.yml +++ b/VirtualMachineService/config/config.yml @@ -1,3 +1,8 @@ +redis: + host: redis + port: 6379 + password: "" + openstack_connection: threads: 30 host: 0.0.0.0 @@ -24,6 +29,7 @@ openstack_connection: availability_zone: default # If you use docker-compose this path needs to be the path you mount the server.pem into certfile: /code/VirtualMachineService/keys/server.pem + use_ssl: True # Used for gateway port calculation , x is always the last octet of the fixed ip (example : (x + 3) *3 ) ) ssh_port_calc_formular: 30000 + x @@ -43,4 +49,4 @@ forc: forc_api_key: github_playbooks_repo: -cloud_site: bielefeld \ No newline at end of file +cloud_site: bielefeld diff --git a/ansible.cfg b/ansible.cfg index 4c66924d..21be49d6 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,8 +1,10 @@ [defaults] host_key_checking = False +record_host_key = False roles_path = ~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:/code/VirtualMachineService/ancon/playbooks/roles [paramiko_connection] host_key_checking = False +record_host_key = False env:ANSIBLE_PARAMIKO_HOST_KEY_CHECKING var: ansible_paramiko_host_key_checking diff --git a/requirements.txt b/requirements.txt index 71da6064..328b0881 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -setuptools==56.2.0 +setuptools==57.0.0 thrift >= 0.11.0,<0.20.0 python-keystoneclient -openstacksdk ==0.56.0 +openstacksdk ==0.57.0 deprecated == 1.2.12 -Click==7.1.2 -ansible==3.4.0 +Click==8.0.1 +ansible==4.0.0 flake8==3.9.2 ruamel.yaml<0.18.0 paramiko==2.7.2