diff --git a/.dependabot/config.yml b/.dependabot/config.yml index 032caf88..21a7f5a6 100644 --- a/.dependabot/config.yml +++ b/.dependabot/config.yml @@ -4,8 +4,16 @@ update_configs: directory: "/" update_schedule: "weekly" target_branch: "dev" + default_reviewers: + - "dweinholz" + - "eKatchko" + - "vktrrdk" - package_manager: "docker" directory: "/" update_schedule: "weekly" target_branch: "dev" + default_reviewers: + - "dweinholz" + - "eKatchko" + - "vktrrdk" diff --git a/.env b/.env new file mode 100644 index 00000000..3aa80436 --- /dev/null +++ b/.env @@ -0,0 +1,11 @@ +CLOUD_CLIENT_TAG=0.1.0-beta.0.8.17 +ELASTIC_USER=elasticboi +FILEBEAT_TAG=7.1.0 +ELASTIC_URL=https://portal-dev.denbi.de:443 + +OS_AUTH_URL=https://openstack.cebitec.uni-bielefeld.de:5000/v3/ +OS_PROJECT_ID=3e552e42945c40aab02af3be9bc67a23 +OS_PROJECT_NAME=portal-pool-dev +OS_USERNAME=portal-user-dev +OS_USER_DOMAIN_NAME=Default +OS_PROJECT_DOMAIN_ID=default diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f81ebad..83693978 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: - name: Publish to Registry uses: elgohr/Publish-Docker-Github-Action@master with: - name: denbicloud/cloud-api + name: denbicloud/cloud-portal-client username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} dockerfile: Dockerfile diff --git a/.gitignore b/.gitignore index c5b9da16..3b235856 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ celerybeat-schedule *.sage.py # Environments +.secrets .env .venv env/ diff --git a/.secrets.in b/.secrets.in new file mode 100644 index 00000000..19f658ee --- /dev/null +++ b/.secrets.in @@ -0,0 +1,3 @@ +OS_PASSWORD= +FORC_API_KEY= +ELASTIC_PASSWORD= diff --git a/Dockerfile b/Dockerfile index 110aefa7..367b7e1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.7-slim +FROM python:3.8.2-slim ADD . /code WORKDIR /code RUN apt-get update -y diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 18d7f6fe..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,16 +0,0 @@ -node { - def image - stage('Clone repository') { - checkout scm - } - - stage('build image'){ - - image = docker.build("denbicloud/cloud-portal-client") - } - stage('push image'){ - withDockerRegistry([ credentialsId: "docker1", url: "" ]) { - image.push("dev") - } - } - } diff --git a/Jenkinsfile-PR b/Jenkinsfile-PR deleted file mode 100644 index 19718c56..00000000 --- a/Jenkinsfile-PR +++ /dev/null @@ -1,11 +0,0 @@ -node { - def image - stage('Clone repository') { - checkout scm - } - - stage('build image'){ - - image = docker.build("denbicloud/cloud-portal-client") - } -} diff --git a/README.md b/README.md index 41a36ee4..da2fd8f6 100644 --- a/README.md +++ b/README.md @@ -171,3 +171,84 @@ if __name__ == '__main__': server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) server.serve() ``` + +### Deployment via ansible + +#### 1.Create your inventory file: + +Example: + +~~~BASH +[test] +REMOTE_IP ansible_user=ubuntu ansible_ssh_private_key_file=PATH_TO_SSH_FILE ansible_python_interpreter=/usr/bin/python3 +~~~ + +where + + * REMOTE_IP is the IP of your staging machine + + * PATH_TO_SSH_FILE is the path to the ssh key of the virtual machine + +#### 2.Set SSH keyforwarding + +In order to checkout the GitHub project you will have to enable +SSH Key forwarding in your `~/.ssh/config` file. + +~~~BASH +Host IP + ForwardAgent yes +~~~ + +where `IP` is the IP of the machine you want to start the portal. + +#### 3.Set SSH Key in Github + +The GitHub repository will cloned using an ssh key you set for the GitHub repository. +You can read how to set an ssh key for the cloud-portal repository on [this website](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/). + + +#### 4.Install needed libraries + +~~~BASH +ansible-galaxy install -r ansible_requirements.yml +~~~ + +#### 5 Create your own secrets file + +Copy the `.secrets.in` to `.secrets`. + +#### 6.Set all variables + +Set all variables that can be found in `.env` and `.secrets` file. + +#### 8.Run the playbook + +You can run the playbook using the following command: + +~~~BASH +ansible-playbook -i inventory_openstack site.yml +~~~ + +where + + * inventory_openstack is your inventory file which you created in the first step. + + +**Choose different files** + +You can also specify different .env , .secrets and server.pem files. + +You can also specify branch, tag, commit that should be checked out with `--extra-vars`. + +For Example: + +~~~BASH +ansible-playbook -i inventory_openstack --extra-vars "repo_version=master" site.yml +~~~ +Optional Keys are: ++ repo_version ++ env_file ++ secrets_file ++ client_server_pem + +**Note:** Default repository is always master. Also by default, the files are taken from the folders as for local start. diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index fb8b82da..2d55a32e 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -168,6 +168,7 @@ def __init__(self, config): try: self.BIBIGRID_URL = cfg["bibigrid"]["bibigrid_url"] self.SUB_NETWORK = cfg["bibigrid"]["sub_network"] + self.BIBIGRID_MODES = cfg["bibigrid"]["bibigrid_modes"] self.logger.info( msg="Bibigrd url loaded: {0}".format(self.BIBIGRID_URL) ) @@ -1697,6 +1698,23 @@ def add_udp_security_group(self, server_id): return True + def detach_ip_from_server(self, server_id, floating_ip): + self.logger.info( + "Detaching floating ip {} from server {}".format(floating_ip, server_id) + ) + try: + self.conn.compute.remove_floating_ip_from_server( + server=server_id, address=floating_ip + ) + return True + except Exception: + self.logger.exception( + "Could not detach floating ip {} from server {}".format( + floating_ip, server_id + ) + ) + return False + def get_servers_by_bibigrid_id(self, bibigrid_id): filters = {"bibigrid_id": bibigrid_id, "name": bibigrid_id} servers = self.conn.list_servers(filters=filters) @@ -1740,20 +1758,33 @@ def get_cluster_status(self, cluster_id): response = req.get( url=request_url, json=body, headers=headers, verify=self.PRODUCTION ) - self.logger.info("Cluster {} status: ".format(cluster_id, response.content)) + self.logger.info("Cluster {} status: {} ".format(cluster_id, response.content)) return response.json() - def get_cluster_info(self, cluster_id): + + def bibigrid_available(self): + if not self.BIBIGRID_URL: + return False + try: + self.get_clusters_info() + return True + except Exception: + self.logger.exception("Bibigrid is offline") + return False + + def get_clusters_info(self): headers = {"content-Type": "application/json"} body = {"mode": "openstack"} request_url = self.BIBIGRID_URL + "list" - self.logger.info(request_url) - response = req.get( url=request_url, json=body, headers=headers, verify=self.PRODUCTION ) self.logger.info(response.json()) infos = response.json()["info"] + return infos + + def get_cluster_info(self, cluster_id): + infos = self.get_clusters_info() for info in infos: self.logger.info(cluster_id) self.logger.info(info) @@ -1802,7 +1833,10 @@ def start_cluster(self, public_key, master_instance, worker_instances, user): "availabilityZone": self.AVAIALABILITY_ZONE, "masterInstance": master_instance, "workerInstances": wI, + "useMasterWithPublicIp": False } + for mode in self.BIBIGRID_MODES: + body.update({mode: True}) request_url = self.BIBIGRID_URL + "create" response = req.post( url=request_url, json=body, headers=headers, verify=self.PRODUCTION diff --git a/VirtualMachineService/VirtualMachineService-remote b/VirtualMachineService/VirtualMachineService-remote index dc433081..668a78e7 100755 --- a/VirtualMachineService/VirtualMachineService-remote +++ b/VirtualMachineService/VirtualMachineService-remote @@ -43,6 +43,8 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' string add_floating_ip_to_server(string openstack_id, string network)') print(' bool create_connection(string username, string password, string auth_url, string user_domain_name, string project_domain_name)') print(' start_server_without_playbook(string flavor, string image, string public_key, string servername, metadata, bool https, bool http, resenv, volume_ids_path_new, volume_ids_path_attach)') + print(' bool bibigrid_available()') + print(' bool detach_ip_from_server(string server_id, string floating_ip)') print(' start_server_with_mounted_volume(string flavor, string image, string public_key, string servername, metadata, bool https, bool http, resenv, volume_ids_path_new, volume_ids_path_attach)') print(' start_server(string flavor, string image, string public_key, string servername, metadata, string diskspace, string volumename, bool https, bool http, resenv)') print(' start_server_with_custom_key(string flavor, string image, string servername, metadata, bool http, bool https, resenv, volume_ids_path_new, volume_ids_path_attach)') @@ -275,6 +277,18 @@ elif cmd == 'start_server_without_playbook': sys.exit(1) pp.pprint(client.start_server_without_playbook(args[0], args[1], args[2], args[3], eval(args[4]), eval(args[5]), eval(args[6]), eval(args[7]), eval(args[8]), eval(args[9]),)) +elif cmd == 'bibigrid_available': + if len(args) != 0: + print('bibigrid_available requires 0 args') + sys.exit(1) + pp.pprint(client.bibigrid_available()) + +elif cmd == 'detach_ip_from_server': + if len(args) != 2: + print('detach_ip_from_server requires 2 args') + sys.exit(1) + pp.pprint(client.detach_ip_from_server(args[0], args[1],)) + elif cmd == 'start_server_with_mounted_volume': if len(args) != 10: print('start_server_with_mounted_volume requires 10 args') diff --git a/VirtualMachineService/VirtualMachineService.py b/VirtualMachineService/VirtualMachineService.py index 002ccd67..8315b5d2 100644 --- a/VirtualMachineService/VirtualMachineService.py +++ b/VirtualMachineService/VirtualMachineService.py @@ -233,6 +233,18 @@ def start_server_without_playbook( """ pass + def bibigrid_available(self): + pass + + def detach_ip_from_server(self, server_id, floating_ip): + """ + Parameters: + - server_id + - floating_ip + + """ + pass + def start_server_with_mounted_volume( self, flavor, @@ -1484,6 +1496,76 @@ def recv_start_server_without_playbook(self): "start_server_without_playbook failed: unknown result", ) + def bibigrid_available(self): + self.send_bibigrid_available() + return self.recv_bibigrid_available() + + def send_bibigrid_available(self): + self._oprot.writeMessageBegin( + "bibigrid_available", TMessageType.CALL, self._seqid + ) + args = bibigrid_available_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_bibigrid_available(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = bibigrid_available_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException( + TApplicationException.MISSING_RESULT, + "bibigrid_available failed: unknown result", + ) + + def detach_ip_from_server(self, server_id, floating_ip): + """ + Parameters: + - server_id + - floating_ip + + """ + self.send_detach_ip_from_server(server_id, floating_ip) + return self.recv_detach_ip_from_server() + + def send_detach_ip_from_server(self, server_id, floating_ip): + self._oprot.writeMessageBegin( + "detach_ip_from_server", TMessageType.CALL, self._seqid + ) + args = detach_ip_from_server_args() + args.server_id = server_id + args.floating_ip = floating_ip + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_detach_ip_from_server(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = detach_ip_from_server_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException( + TApplicationException.MISSING_RESULT, + "detach_ip_from_server failed: unknown result", + ) + def start_server_with_mounted_volume( self, flavor, @@ -3218,6 +3300,10 @@ def __init__(self, handler): self._processMap[ "start_server_without_playbook" ] = Processor.process_start_server_without_playbook + self._processMap["bibigrid_available"] = Processor.process_bibigrid_available + self._processMap[ + "detach_ip_from_server" + ] = Processor.process_detach_ip_from_server self._processMap[ "start_server_with_mounted_volume" ] = Processor.process_start_server_with_mounted_volume @@ -3834,6 +3920,58 @@ def process_start_server_without_playbook(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_bibigrid_available(self, seqid, iprot, oprot): + args = bibigrid_available_args() + args.read(iprot) + iprot.readMessageEnd() + result = bibigrid_available_result() + try: + result.success = self._handler.bibigrid_available() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException( + TApplicationException.INTERNAL_ERROR, "Internal error" + ) + oprot.writeMessageBegin("bibigrid_available", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_detach_ip_from_server(self, seqid, iprot, oprot): + args = detach_ip_from_server_args() + args.read(iprot) + iprot.readMessageEnd() + result = detach_ip_from_server_result() + try: + result.success = self._handler.detach_ip_from_server( + args.server_id, args.floating_ip + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException( + TApplicationException.INTERNAL_ERROR, "Internal error" + ) + oprot.writeMessageBegin("detach_ip_from_server", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_start_server_with_mounted_volume(self, seqid, iprot, oprot): args = start_server_with_mounted_volume_args() args.read(iprot) @@ -8474,6 +8612,287 @@ def __ne__(self, other): ) +class bibigrid_available_args(object): + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("bibigrid_available_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(bibigrid_available_args) +bibigrid_available_args.thrift_spec = () + + +class bibigrid_available_result(object): + """ + Attributes: + - success + + """ + + def __init__( + self, success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("bibigrid_available_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(bibigrid_available_result) +bibigrid_available_result.thrift_spec = ((0, TType.BOOL, "success", None, None,),) # 0 + + +class detach_ip_from_server_args(object): + """ + Attributes: + - server_id + - floating_ip + + """ + + def __init__( + self, server_id=None, floating_ip=None, + ): + self.server_id = server_id + self.floating_ip = floating_ip + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.server_id = ( + iprot.readString().decode("utf-8") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.floating_ip = ( + iprot.readString().decode("utf-8") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("detach_ip_from_server_args") + if self.server_id is not None: + oprot.writeFieldBegin("server_id", TType.STRING, 1) + oprot.writeString( + self.server_id.encode("utf-8") + if sys.version_info[0] == 2 + else self.server_id + ) + oprot.writeFieldEnd() + if self.floating_ip is not None: + oprot.writeFieldBegin("floating_ip", TType.STRING, 2) + oprot.writeString( + self.floating_ip.encode("utf-8") + if sys.version_info[0] == 2 + else self.floating_ip + ) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(detach_ip_from_server_args) +detach_ip_from_server_args.thrift_spec = ( + None, # 0 + (1, TType.STRING, "server_id", "UTF8", None,), # 1 + (2, TType.STRING, "floating_ip", "UTF8", None,), # 2 +) + + +class detach_ip_from_server_result(object): + """ + Attributes: + - success + + """ + + def __init__( + self, success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("detach_ip_from_server_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(detach_ip_from_server_result) +detach_ip_from_server_result.thrift_spec = ( + (0, TType.BOOL, "success", None, None,), # 0 +) + + class start_server_with_mounted_volume_args(object): """ Attributes: diff --git a/VirtualMachineService/config/config.yml b/VirtualMachineService/config/config.yml index 69c401c9..58a5172b 100644 --- a/VirtualMachineService/config/config.yml +++ b/VirtualMachineService/config/config.yml @@ -32,6 +32,9 @@ bibigrid: # Url for Bibigrid API bibigrid_url: https://172.21.0.1:8443/bibigrid/ sub_network: portalexternalsubnetwork + bibigrid_modes: + - slurm + forc: forc_url: https://proxy-dev.bi.denbi.de:5000/ forc_allowed: diff --git a/VirtualMachineService/keys/localhost/server.pem b/VirtualMachineService/keys/localhost/server.pem index e0382c3e..355ab421 100644 --- a/VirtualMachineService/keys/localhost/server.pem +++ b/VirtualMachineService/keys/localhost/server.pem @@ -1,45 +1,49 @@ -----BEGIN CERTIFICATE----- -MIICpDCCAYwCCQDtWP54D1EuLTANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls -b2NhbGhvc3QwHhcNMTgwNDI1MTMwOTM3WhcNMjYwNzEyMTMwOTM3WjAUMRIwEAYD -VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDz -nDzlJUUtV9mVKwzVF197DALocSnWD2JB14s+EGPCB6qKqfV/gCfzCNehRWKr1Bot -2o+SepZgmBvT4uM7iT/W4vH28NY3FlPPewqWoKJlBaRysdmOHGQWndJCZkkcDuSG -A+dHRS3dy2maNrK0dmfYAZx08N0AErHP2v1emoWw2MSoKM+A1vuJ+T/s5oMrkDaU -Y8XbldjoKFijfUFk7NJPKFwrGnW8n1ogR6yOnDa1SuX2r895WkX5Whqej4wzcThq -IRgltWNDujc9CWEfgq7yDMvEg2aB/I580pOpyTHWX1ysIzl5acKIxYzXaImpzvXi -99lrjxhUmWnqQrmTcHwZAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAIutV/jdZAWa -2xi+BrKxONDJ/r/DdchRG5dBuW42N3fr8jENtRf2/h1PwuxduMix1Wo2WApYbrpe -tOQpzM1So9ZdL6Ba/r5JObPiWfXiWxnRFTAsTBJYMs5F1uuywvBSAf1pRLg4NHSg -VRRpG3znZTbV8GjbgVJQxoz/iPMO9nPt7sPXT7zsGUhLjn8AIsMMLJZ9ZNJWGt9X -x5n6qbtWVCxlAkR+uFFzESmMckv/nMZHCSk3Qs7+aMCrd2L2C4vrXUaxE3v86T5O -DZ1eY4s2suyKZvpz21rUykmWP3XYTDfaaCBVPF7zqWevQUjQln5dnYOHnR28o252 -1/sEn+oeyFA= +MIIDazCCAlOgAwIBAgIUH6tDIIYG2zDcJ4C8WXYFF3QAC2EwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCREUxCjAIBgNVBAgMAS4xCjAIBgNVBAcMAS4xCjAIBgNV +BAoMAS4xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMDA0MTQxMTI4MDlaFw0yODA3 +MDExMTI4MDlaMEUxCzAJBgNVBAYTAkRFMQowCAYDVQQIDAEuMQowCAYDVQQHDAEu +MQowCAYDVQQKDAEuMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCoID2YCnbfQeWViuwruBSa9U0Ww8eKx26zjtt/OOCT +IBPdBI+qL3LNh6oXh1gA5dZEv1FvOm7ngNGb0XC2zDY0h+7j+w7ctnRI0qhfXA7A +xSLDfsYgOZssDXCJd9FsQT4XoGr+YxFha8uryTQf96LMHo44e06CT8QQqW7g5Bbw +vKXHcs4M3h3a7QPizEh4gZuEgpj4UojnTWQLqHzr95TmNMZDYNBmTsugayFmFeOX +WdJaKxLFTIAFkInuuKT/UZP3X5qOkGokx696aSW5wqZXzJYDVh2WTnwROd6rvhvt +icpfwCh2VfaSVnBTlOjN+uFTm91yityXnXHcQu+Ou/ItAgMBAAGjUzBRMB0GA1Ud +DgQWBBSUJQ8O6VEdaZqXEONCc/QiAY1/tTAfBgNVHSMEGDAWgBSUJQ8O6VEdaZqX +EONCc/QiAY1/tTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCQ +UxPsCjWoe5PGyxLEt9awUum/8EnBKLLw2xGOZjuSljdNJDLCG1J4rCGw7eqcg66Z +gVO6+8EcVsohd2TJEophivqCxhe//phnAeB+lYt8gB0pX0ahgQ5xk0/TeZxNvOSg +DMq9ZWSmqB7n3P7LsAN4XVZ/TSfk5O3aC7O7SVQsV8Rp3P3w4OXu5F5ZSyDwj2MX +gEynz1suoCnV8G1nnUkv6Il96YbWZAafWe4mDhMiTRKQtaM/7Y3l8B/lHsBoD5i9 +WsRdO+Hwn+i9AiKILhB/Nlwze3LDVuWp0QIg9dtd8DQdjPRU8aPqiNKaWb3RN8V4 +phO3KSJlQJr05+QYJVGv -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDznDzlJUUtV9mV -KwzVF197DALocSnWD2JB14s+EGPCB6qKqfV/gCfzCNehRWKr1Bot2o+SepZgmBvT -4uM7iT/W4vH28NY3FlPPewqWoKJlBaRysdmOHGQWndJCZkkcDuSGA+dHRS3dy2ma -NrK0dmfYAZx08N0AErHP2v1emoWw2MSoKM+A1vuJ+T/s5oMrkDaUY8XbldjoKFij -fUFk7NJPKFwrGnW8n1ogR6yOnDa1SuX2r895WkX5Whqej4wzcThqIRgltWNDujc9 -CWEfgq7yDMvEg2aB/I580pOpyTHWX1ysIzl5acKIxYzXaImpzvXi99lrjxhUmWnq -QrmTcHwZAgMBAAECggEAe8X8IqjnFvy1rlarNHGlr40QxS0KBprWX4G2W5hUNzdR -7RYhh+CDa7YoaOYqFge0LWwI2nGyie1+5g83SP6ZEwNH0SHYlTCg8KAhZxrCp/VP -8n5ufQfrypnxRI9XwgvHDFhbYBjEDxbU1ufHsG0god1fswLWJr1Y3rTzWufD2W+r -37MMfOdyHK4WHTewWJYY/YX68fFyELt15NQHnz7uQJznY7Wwv/norX1jZsksy9VE -m8geLVIHdJOIyaHy+aAsMzWQ30rk8YU/7gNN3FanYdKv/WA/9qstsg61xYoH5y4Q -igKDkuW3h2ra4u2mLLR1KZEHEQYAwgtIMjL7WsPqAQKBgQD9hH+ZPZAW/NDSJ/bX -dhMqiRgwB3Ak/OnOmcgsDuroT3xtdUT1xBq21NzUWoiW6Eni/PF/9QxuIm//kMQk -uKk8AQyGM3RDtk6p69Kd+P9DfLb/EViHdug+/X3g1M378X2InLQ39cF8963+MOkA -kCJOQqyYWGRoz1tFCHSHEjjbuQKBgQD1/uePuPPGrJ3GS8XCIB6jy7TwqDbU+gJa -AAlHznn9zpDycSe2otYYsn/jSdRmbJA1K82UYE57o3kgImyDLPPyumhLHdE2db04 -Qt+sAwlJuBVIN5oIF0eCxJE0owKEv/LSQrx5k7NDNJnC6oTRByQhMOQmuDE81AvE -++u1NIqTYQKBgCW4TB/pyAGTliJerdx7BfkDMuUahjYKp2MFAIKFqa/s+69xjlQe -3HatjznNSpthbnTme6w1MqdEWnOk/3oTO6Lg6RMinr7gZi3kF49Hl/tbmAK1ev2U -1JNithL4trkwwiPMZQgaebJ1S58ReemKiIAqbXGxXSc3CvZnfqcGMA2JAoGBAOcF -2ZqUa/ytqA7Ooo7Jc30IlolcsQIbOZIkbpLNSqweDuphcpaHHuJa4NgoQh1e35/P -32X1fD+JKvwO2DHpScEvZzsEabLnqCjqRP1abKKbCwGjljj7A+G/kU8JuBfL3NOy -cScMm1IGWFFBxjI4rqmtLNdgLH2f9t2saZToHyyBAoGBANzB+jBJtA/AUn6VteTF -ceU0wpzyINz1nh/V489puhcVfz3s9+TE+zN2V4DOVF6yOm3hMOQuLyEWhK4xCrVq -EAYhXBm9K5EEQDE160QbJBjQMYXBsDJ4M/O2kj1dt1lMos2COIj0XdDtFIo4tj4b -9kx91hF7mrsoD4KyktijPGf4 +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCoID2YCnbfQeWV +iuwruBSa9U0Ww8eKx26zjtt/OOCTIBPdBI+qL3LNh6oXh1gA5dZEv1FvOm7ngNGb +0XC2zDY0h+7j+w7ctnRI0qhfXA7AxSLDfsYgOZssDXCJd9FsQT4XoGr+YxFha8ur +yTQf96LMHo44e06CT8QQqW7g5BbwvKXHcs4M3h3a7QPizEh4gZuEgpj4UojnTWQL +qHzr95TmNMZDYNBmTsugayFmFeOXWdJaKxLFTIAFkInuuKT/UZP3X5qOkGokx696 +aSW5wqZXzJYDVh2WTnwROd6rvhvticpfwCh2VfaSVnBTlOjN+uFTm91yityXnXHc +Qu+Ou/ItAgMBAAECggEAbrXsWXZqmNG38SWuJ4vsZa4quVbqy4RQbYUhHUHDAJ8a +9WvPfu4H1zgC1cOWO+Edm9+wLGJ6LZAydz0P0A/hke4BbbchLk4nfNzbaxAf1uj5 +32pX5ViJyhR/isaP7Y146e4FBscN5u0pDVJNNM/JMEjetmrcZAtKvP4VuZn5UUSX +Wom8hVaScQqa3zZkm0hlQZ0UaQNffw3PkkYVnOnZtUV1nHJAjAE2gpR59GfPYzHv +g1Ag+DwFKk2ReRqoiR+E0VXtQPIUOuiraV9edcjOz/gZm7L1IRvKF81v7btr02v+ +zZFkRRGkCmGO5+TCBmJ0ufP+z1fND1ruyFFGNWKbsQKBgQDQYcsH0GKmFYJ0/rDX +5KBj/rKzr4o78W9kS6jVZW2SGSHnajB5TpYrZXk3bCQjoAwD4QrWiEsoqFNmTmFu +XD1VyMy/+Ft2BS9zZpnNP0qWHNPEXj4kzAZMjuUeXZIocw9wQzMtz67DIgnZ/YIR +US/LKE81FGheGWBSSBPdcoz9SwKBgQDOi36u6tD8AkWjW3+Si8hYzQ2Te18cqOjy +GyCpGPeMBklbN/VxFcPbpWTrd0nIECmzTB3EFAV2LEWilAyCI/HfmPoUFghoz6yC +HCVJ+Ux+2Fd1xZLgllwlPMODtZTj5WWTsJZ1mB7Oefh88FSaoW0hJxhnhftpc5XC +DEtEqI17ZwKBgC9eq7rBAkvjA974rtbFIDjbA2CZfQEXZ00TbfPktL3LfMuQdih9 +VuHSlXWl3alDjrerBr2yCfPaH+tX0go2abMAjJrvKEtx7lF7d2cTBi+Nqg4vpkzT +HHN24FrAxqLvVQkeHZpCZeXx7O9jE5hALm8uYKMeWoNZnW3W+L0Udzx1AoGBALLy +UJOjheSQ41ygzWpeCPpGzdhVLxZZkVkQ/78BjpeWar/JP7D2NnZ6WXFYLGhU+IwL +Ck7zObgkq0AsHJo0Ij6i/yef3zFPnBot4HCyuP+82CmFP7etDbCFcK8QBytbeCH5 +AKJpZWdmbU9xuRsveKkcghxPDH9UUQ+KLzwQ/GNJAoGBALyQhQu1r/Op53rXxhrm ++X8IKWW7Qak7O1Qp4pF/qSlKQpuUo2zdvmqleKmWWPMGDvAhloyo/TCxD82nTXJW +C1RcYuSTwOw22RUIp683fcV7N13Jm63EEsH0hDMq2mvDyXbu5yKYNIdlPgCxUEzx +zsk41o4jgpEFfJQDB+mQ7ZgD -----END PRIVATE KEY----- diff --git a/ansible_requirements.yml b/ansible_requirements.yml new file mode 100644 index 00000000..d1c08e70 --- /dev/null +++ b/ansible_requirements.yml @@ -0,0 +1,2 @@ +- src: geerlingguy.docker +- src: geerlingguy.pip diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 723256d3..2ab95dae 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -6,6 +6,9 @@ services: dockerfile: Dockerfile ports: - "9090:9090" + + env_file: + - .env environment: - OS_AUTH_URL diff --git a/docker-compose.yml b/docker-compose.yml index b25d6c6b..d5741165 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,9 @@ services: image: denbicloud/cloud-portal-client:${CLOUD_CLIENT_TAG} ports: - "9090:9090" + + env_file: + - .env environment: - OS_AUTH_URL @@ -16,12 +19,12 @@ services: - FORC_API_KEY volumes: - - ./VirtualMachineService/config/config.yml:/code/VirtualMachineService/config.yml - - ./VirtualMachineService/keys/localhost/server.pem:/code/VirtualMachineService/keys/server.pem + - ${client_PERSISTENT_PATH}/config.yml:/code/VirtualMachineService/config.yml + - ${client_PERSISTENT_PATH}/server.pem:/code/VirtualMachineService/keys/server.pem - ./logs/client:/code/VirtualMachineService/log:rw command: python3 VirtualMachineServer.py /code/VirtualMachineService/config.yml networks: - - portal + - portal redis: image: redis:5.0.5 diff --git a/plays/.general_variables.yml.kate-swp b/plays/.general_variables.yml.kate-swp new file mode 100644 index 00000000..a394e7ba Binary files /dev/null and b/plays/.general_variables.yml.kate-swp differ diff --git a/plays/general_variables.yml b/plays/general_variables.yml new file mode 100644 index 00000000..7e434fd9 --- /dev/null +++ b/plays/general_variables.yml @@ -0,0 +1,6 @@ +client: + PERSISTENT_PATH: /home/ubuntu/persistent + CONFIG_DIR_PATH: /home/ubuntu/client/VirtualMachineService/config + CERT_FILE_PATH: /home/ubuntu/client/VirtualMachineService/keys + REPO_PATH: /home/ubuntu/client + diff --git a/plays/setup_basics.yml b/plays/setup_basics.yml new file mode 100644 index 00000000..babc1b15 --- /dev/null +++ b/plays/setup_basics.yml @@ -0,0 +1,52 @@ +- name: Setup and install Basics + hosts: all + become: yes + vars_files: + - general_variables.yml + + roles: + - role: geerlingguy.docker + + pre_tasks: + - name: Verify Ansible version. + assert: + that: "ansible_version.full is version_compare('2.7', '>=')" + msg: > + "You must update Ansible to at least 2.7"L + + tasks: + - name: Update cache + apt: + upgrade: yes + update_cache: yes + cache_valid_time: 86400 #One day + + - name: Checkout cloud-portal-client repository + git: + repo: "git@github.com:deNBI/cloud-portal-client.git" + dest: "{{ client.REPO_PATH }}" + accept_hostkey: yes + force: yes + version: "{{ repo_version | default('master') }}" + become: no + + - name: Install python3-pip + apt: + state: latest + name: python3-pip + + - name: Install list of packages + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - libffi-dev + - libssl-dev + - jq + + - name: Install docker-compose with pip + pip: + name: docker-compose + + diff --git a/plays/setup_client.yml b/plays/setup_client.yml new file mode 100644 index 00000000..8f89abe6 --- /dev/null +++ b/plays/setup_client.yml @@ -0,0 +1,112 @@ +- name: Setup Client + hosts: all + become: yes + vars_files: + - general_variables.yml + tasks: + + - name: Setup persistent client directory + file: + path: "{{ client.PERSISTENT_PATH }}" + state: directory + - name: Setup persistent client cert directory + file: + path: "{{ client.PERSISTENT_PATH }}" + state: directory + + - name: Check if client config file exists already + stat: + path: "{{ client.PERSISTENT_PATH }}/config.yml" + register: stat_client_config + + - name: Check if server pem file exists already + stat: + path: "{{ client.PERSISTENT_PATH }}/server.pem" + register: stat_client_pem + + - name: Copy client config file from config folder + copy: + remote_src: True + src: "{{ client.CONFIG_DIR_PATH }}/config.yml" + dest: "{{ client.PERSISTENT_PATH }}/config.yml" + when: client_config is not defined and stat_client_config.stat.exists == False + + - name: Copy specified client config file to remote machine + copy: src={{ item.src }} dest={{ item.dest }} + with_items: + - { src: "{{client_config}}", dest: "{{ client.PERSISTENT_PATH }}/config.yml" } + when: client_config is defined + + + - name: Copy specified server pem file to remote machine persistent + copy: src={{ item.src }} dest={{ item.dest }} + with_items: + - { src: "{{client_server_pem}}", dest: "{{ client.PERSISTENT_PATH }}/server.pem" } + when: client_server_pem is defined + + - name: Copy specified server pem file to remote machine persistent from default path + copy: src={{ item.src }} dest={{ item.dest }} + with_items: + - { src: "{{client.CERT_FILE_PATH}}", dest: "{{ client.PERSISTENT_PATH }}/server.pem" } + when: client_server_pem is not defined and stat_client_pem.exists == False + + + - name: Copy env file default + copy: + src: "../.env" + dest: "{{ client.REPO_PATH }}/.env" + backup: yes + when: env_file is not defined + + - name: Copy env file + copy: + src: "{{env_file}}" + dest: "{{ client.REPO_PATH }}/.env" + backup: yes + when: env_file is defined + + + - name: Copy secrets file default + copy: + src: "../.secrets" + dest: "{{ client.REPO_PATH }}/.secrets" + backup: yes + when: secrets_file is not defined + + - name: Copy secrets file + copy: + src: "{{secrets_file}}" + dest: "{{ client.REPO_PATH }}/.secrets" + backup: yes + when: secrets_file is defined + + + + - name: Create variable files + copy: + content: "{{ item.content | dict2items | map('to_json') | map('regex_replace', '\"key\":\\s\"(.*)\"', lookup('vars', 'regex_env')) | map('from_json') | list}}" + dest: "{{ client.REPO_PATH }}/.ansible_environment_{{ item.name }}.json" + backup: yes + vars: + regex_env: "\"key\": \"{{ item.name }}_\\1\"" + with_items: + - { content: "{{ client }}", name: 'client' } + + - name: Transform json to properties file + shell: rm -f "{{ client.REPO_PATH }}"/.ansible_environment && jq -r '.[]|"\(.key)=\(.value)"' "{{ client.REPO_PATH }}"/.ansible_environment_*.json >> "{{ client.REPO_PATH }}"/.ansible_environment + + - name: Append secrets to env file + shell: cat "{{ client.REPO_PATH }}"/.secrets "{{ client.REPO_PATH }}"/.ansible_environment >> "{{ client.REPO_PATH }}"/.env + + + + + - name: Start docker container + docker_compose: + pull: yes + project_src: "{{ client.REPO_PATH }}" + files: + - docker-compose.yml + become: yes + + diff --git a/portal_client.thrift b/portal_client.thrift index 32d65ab9..da7dc64e 100644 --- a/portal_client.thrift +++ b/portal_client.thrift @@ -368,7 +368,8 @@ service VirtualMachineService { throws (1:nameException e,2:ressourceException r,3:serverNotFoundException s,4: networkNotFoundException n,5:imageNotFoundException i,6:flavorNotFoundException f,7:otherException o) - + bool bibigrid_available() + bool detach_ip_from_server(1:string server_id,2:string floating_ip) map start_server_with_mounted_volume( /** Name of the Flavor to use.*/ diff --git a/requirements.txt b/requirements.txt index 04892fdb..6ea0a90e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ setuptools==46.1.3 thrift >= 0.11.0,<0.20.0 python-keystoneclient -openstacksdk ==0.43.0 -deprecated == 1.2.7 -Click==7.0 -flake8==3.7.7 +openstacksdk ==0.46.0 +deprecated == 1.2.9 +Click==7.1.1 +flake8==3.7.9 ansible==2.9.6 ruamel.yaml<0.17.0 paramiko==2.7.1 pyvim==3.0.2 redis==3.4.1 -requests==2.22.0 +requests==2.23.0 diff --git a/gateway/gateway_TCP.sh b/scripts/gateway/gateway_TCP.sh similarity index 100% rename from gateway/gateway_TCP.sh rename to scripts/gateway/gateway_TCP.sh diff --git a/gateway/gateway_UDP.sh b/scripts/gateway/gateway_UDP.sh similarity index 100% rename from gateway/gateway_UDP.sh rename to scripts/gateway/gateway_UDP.sh diff --git a/scripts/generate_new_test_pems.sh b/scripts/generate_new_test_pems.sh new file mode 100644 index 00000000..fddf37fe --- /dev/null +++ b/scripts/generate_new_test_pems.sh @@ -0,0 +1,19 @@ +#!/usr/bin/expect +echo Creating new dir "new_pem" +mkdir new_pem +cd new_pem +openssl req -new -x509 -nodes -days 3000 -out server.crt -keyout server.key -subj "/C=DE/ST=./L=./O=./CN=$1" + +openssl x509 -in server.crt -text > CA.pem +cat server.crt server.key > server.pem +openssl pkcs12 -export -clcerts -in server.crt -inkey server.key -out server.p12 -passout pass:thrift +openssl genrsa -out client.key +openssl req -new -nodes -key client.key -out client.csr -subj "/C=DE/ST=./L=./O=./CN=$1" -passin pass:thrift +openssl x509 -req -days 3000 -in client.csr -CA CA.pem -CAkey server.key -set_serial 01 -out client.crt +openssl pkcs12 -export -clcerts -in client.crt -inkey client.key -out client.p12 -passout pass:thrift +openssl pkcs12 -in client.p12 -out client.pem -clcerts -passin pass:thrift -passout pass:thrift +openssl rsa -in client.pem -out client_no_pass.pem -passin pass:thrift +sed -i '/-----BEGIN ENCRYPTED PRIVATE KEY-----/Q' client.pem +cat client_no_pass.pem >> client.pem +find . -type f -not -name '*.pem' -print0 | xargs -0 rm +rm client_no_pass.pem diff --git a/site.yml b/site.yml new file mode 100644 index 00000000..f5602719 --- /dev/null +++ b/site.yml @@ -0,0 +1,2 @@ +- import_playbook: plays/setup_basics.yml +- import_playbook: plays/setup_client.yml