From e7570b121543044dad41ee7bf7a07183de2ce34f Mon Sep 17 00:00:00 2001 From: David Weinholz Date: Mon, 1 Jul 2019 11:39:41 +0200 Subject: [PATCH 01/18] feat(Snapshot):addeed description --- .../VirtualMachineHandler.py | 14 ++++++---- .../VirtualMachineService-remote | 8 +++--- .../VirtualMachineService.py | 27 ++++++++++++++----- portal_client.thrift | 4 ++- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 9500a196..8100f2bf 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -744,7 +744,7 @@ def get_ip_ports(self, openstack_id): ) return {} - def create_snapshot(self, openstack_id, name, elixir_id, base_tag): + def create_snapshot(self, openstack_id, name, elixir_id, base_tag, description): """ Create an Snapshot from an server. @@ -770,17 +770,21 @@ def create_snapshot(self, openstack_id, name, elixir_id, base_tag): try: snapshot = self.conn.get_image_by_id(snapshot_munch["id"]) snapshot_id = snapshot["id"] - #todo check again + # todo check again try: + image = self.conn.get_image(name_or_id=snapshot_id) + if description: + self.conn.update_image_properties( + image=image, + meta={'description': description}) + self.conn.image.add_tag( image=snapshot_id, tag="snapshot_image:{0}".format(base_tag) ) except Exception: self.logger.exception("Tag error catched") pass - try : - self.logger.exception("Tag error catched") - + try: self.conn.image.add_tag(image=snapshot_id, tag=elixir_id) except Exception: pass diff --git a/VirtualMachineService/VirtualMachineService-remote b/VirtualMachineService/VirtualMachineService-remote index 1376ab87..d0420328 100755 --- a/VirtualMachineService/VirtualMachineService-remote +++ b/VirtualMachineService/VirtualMachineService-remote @@ -40,7 +40,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' bool add_security_group_to_server(bool http, bool https, bool udp, string server_id)') print(' VM get_server(string openstack_id)') print(' bool stop_server(string openstack_id)') - print(' string create_snapshot(string openstack_id, string name, string elixir_id, string base_tag)') + print(' string create_snapshot(string openstack_id, string name, string elixir_id, string base_tag, string description)') print(' get_limits()') print(' bool delete_image(string image_id)') print(' bool delete_volume_attachment(string volume_id, string server_id)') @@ -227,10 +227,10 @@ elif cmd == 'stop_server': pp.pprint(client.stop_server(args[0],)) elif cmd == 'create_snapshot': - if len(args) != 4: - print('create_snapshot requires 4 args') + if len(args) != 5: + print('create_snapshot requires 5 args') sys.exit(1) - pp.pprint(client.create_snapshot(args[0], args[1], args[2], args[3],)) + pp.pprint(client.create_snapshot(args[0], args[1], args[2], args[3], args[4],)) elif cmd == 'get_limits': if len(args) != 0: diff --git a/VirtualMachineService/VirtualMachineService.py b/VirtualMachineService/VirtualMachineService.py index 58c6f81e..c6da4bcf 100644 --- a/VirtualMachineService/VirtualMachineService.py +++ b/VirtualMachineService/VirtualMachineService.py @@ -199,7 +199,7 @@ def stop_server(self, openstack_id): """ pass - def create_snapshot(self, openstack_id, name, elixir_id, base_tag): + def create_snapshot(self, openstack_id, name, elixir_id, base_tag, description): """ Create Snapshot. Returns: Id of new Snapshot @@ -210,6 +210,7 @@ def create_snapshot(self, openstack_id, name, elixir_id, base_tag): - name: Name of new Snapshot - elixir_id: Elixir-Id of the user who requested creation of Snapshot - base_tag: Tag with which the servers image is also tagged ( for connection information at the webapp) + - description: Description of the new snapshot """ pass @@ -950,7 +951,7 @@ def recv_stop_server(self): raise result.e raise TApplicationException(TApplicationException.MISSING_RESULT, "stop_server failed: unknown result") - def create_snapshot(self, openstack_id, name, elixir_id, base_tag): + def create_snapshot(self, openstack_id, name, elixir_id, base_tag, description): """ Create Snapshot. Returns: Id of new Snapshot @@ -961,18 +962,20 @@ def create_snapshot(self, openstack_id, name, elixir_id, base_tag): - name: Name of new Snapshot - elixir_id: Elixir-Id of the user who requested creation of Snapshot - base_tag: Tag with which the servers image is also tagged ( for connection information at the webapp) + - description: Description of the new snapshot """ - self.send_create_snapshot(openstack_id, name, elixir_id, base_tag) + self.send_create_snapshot(openstack_id, name, elixir_id, base_tag, description) return self.recv_create_snapshot() - def send_create_snapshot(self, openstack_id, name, elixir_id, base_tag): + def send_create_snapshot(self, openstack_id, name, elixir_id, base_tag, description): self._oprot.writeMessageBegin('create_snapshot', TMessageType.CALL, self._seqid) args = create_snapshot_args() args.openstack_id = openstack_id args.name = name args.elixir_id = elixir_id args.base_tag = base_tag + args.description = description args.write(self._oprot) self._oprot.writeMessageEnd() self._oprot.trans.flush() @@ -1847,7 +1850,7 @@ def process_create_snapshot(self, seqid, iprot, oprot): iprot.readMessageEnd() result = create_snapshot_result() try: - result.success = self._handler.create_snapshot(args.openstack_id, args.name, args.elixir_id, args.base_tag) + result.success = self._handler.create_snapshot(args.openstack_id, args.name, args.elixir_id, args.base_tag, args.description) msg_type = TMessageType.REPLY except TTransport.TTransportException: raise @@ -4550,15 +4553,17 @@ class create_snapshot_args(object): - name: Name of new Snapshot - elixir_id: Elixir-Id of the user who requested creation of Snapshot - base_tag: Tag with which the servers image is also tagged ( for connection information at the webapp) + - description: Description of the new snapshot """ - def __init__(self, openstack_id=None, name=None, elixir_id=None, base_tag=None,): + def __init__(self, openstack_id=None, name=None, elixir_id=None, base_tag=None, description=None,): self.openstack_id = openstack_id self.name = name self.elixir_id = elixir_id self.base_tag = base_tag + self.description = description def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -4589,6 +4594,11 @@ def read(self, iprot): self.base_tag = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.description = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -4615,6 +4625,10 @@ def write(self, oprot): oprot.writeFieldBegin('base_tag', TType.STRING, 4) oprot.writeString(self.base_tag.encode('utf-8') if sys.version_info[0] == 2 else self.base_tag) oprot.writeFieldEnd() + if self.description is not None: + oprot.writeFieldBegin('description', TType.STRING, 5) + oprot.writeString(self.description.encode('utf-8') if sys.version_info[0] == 2 else self.description) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -4638,6 +4652,7 @@ def __ne__(self, other): (2, TType.STRING, 'name', 'UTF8', None, ), # 2 (3, TType.STRING, 'elixir_id', 'UTF8', None, ), # 3 (4, TType.STRING, 'base_tag', 'UTF8', None, ), # 4 + (5, TType.STRING, 'description', 'UTF8', None, ), # 5 ) diff --git a/portal_client.thrift b/portal_client.thrift index 11d01bf2..42a75b55 100644 --- a/portal_client.thrift +++ b/portal_client.thrift @@ -364,7 +364,9 @@ service VirtualMachineService { 3: string elixir_id, /** Tag with which the servers image is also tagged ( for connection information at the webapp) */ - 4:string base_tag) + 4:string base_tag, + /** Description of the new snapshot*/ + 5:string description) throws (1:serverNotFoundException e), From e6a31487e792db33b39cb15bba1e3178a7e50cdb Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 12:43:25 +0200 Subject: [PATCH 02/18] feature(ansible-bioconda): ansible running playbook working --- Dockerfile | 1 + .../VirtualMachineHandler.py | 152 +++++++++--- VirtualMachineService/VirtualMachineServer.py | 4 +- .../VirtualMachineService-remote | 7 + .../VirtualMachineService.py | 221 ++++++++++++++++++ .../ancon/BiocondaPlaybook.py | 66 ++++++ VirtualMachineService/ancon/callback.py | 199 ++++++++++++++++ .../ancon/playbooks/bioconda.yml | 35 +++ .../ancon/playbooks/variables.yml | 8 + ansible.cfg | 7 + portal_client.thrift | 10 +- requirements.txt | 8 +- 12 files changed, 674 insertions(+), 44 deletions(-) create mode 100644 VirtualMachineService/ancon/BiocondaPlaybook.py create mode 100644 VirtualMachineService/ancon/callback.py create mode 100644 VirtualMachineService/ancon/playbooks/bioconda.yml create mode 100644 VirtualMachineService/ancon/playbooks/variables.yml create mode 100644 ansible.cfg diff --git a/Dockerfile b/Dockerfile index 16bc6c91..5974be72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,4 +2,5 @@ FROM python:3.6.6-slim ADD . /code WORKDIR /code RUN pip install -r requirements.txt +COPY ansible.cfg /etc/ansible/ WORKDIR /code/VirtualMachineService diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index e4bfabf2..8e985751 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -3,6 +3,13 @@ Which can be used for the PortalClient. """ +from tempfile import NamedTemporaryFile, TemporaryDirectory +import shutil +import ruamel.yaml +import subprocess +import shlex +import sys +from VirtualMachineService.ancon.BiocondaPlaybook import BiocondaPlaybook try: from VirtualMachineService import Iface @@ -42,7 +49,6 @@ import base64 from oslo_utils import encodeutils - class VirtualMachineHandler(Iface): """Handler which the PortalClient uses.""" @@ -64,7 +70,7 @@ def create_connection(self): ) conn.authorize() except Exception as e: - self.logger.error("Client failed authentication at Openstack : {0}", e) + self.logger.exception("Client failed authentication at Openstack : {0}", e) raise authenticationException( Reason="Client failed authentication at Openstack" ) @@ -162,7 +168,7 @@ def findUser(keystone, name): keystone.users.update(user, password=password) return password except Exception as e: - self.logger.error( + self.logger.exception( "Set Password for user {0} failed : {1}".format(user, str(e)) ) return otherException(Reason=str(e)) @@ -191,7 +197,7 @@ def get_Flavors(self): flavors.append(flavor) return flavors except Exception as e: - self.logger.error("Get Flavors Error: {0}".format(e)) + self.logger.exception("Get Flavors Error: {0}".format(e)) return () @deprecated( @@ -215,7 +221,7 @@ def check_Version(self, version): else: return False except Exception as e: - self.logger.error("Compare Version Error: {0}".format(e)) + self.logger.exception("Compare Version Error: {0}".format(e)) return False def get_client_version(self): @@ -264,7 +270,7 @@ def get_Images(self): return images except Exception as e: - self.logger.error("Get Images Error: {0}".format(e)) + self.logger.exception("Get Images Error: {0}".format(e)) return () def get_Image_with_Tag(self, id): @@ -294,7 +300,7 @@ def get_Image_with_Tag(self, id): ) return image except Exception as e: - self.logger.error("Get Image {0} with Tag Error: {1}".format(id, e)) + self.logger.exception("Get Image {0} with Tag Error: {1}".format(id, e)) return def import_keypair(self, keyname, public_key): @@ -323,7 +329,7 @@ def import_keypair(self, keyname, public_key): return keypair return keypair except Exception as e: - self.logger.error("Import Keypair {0} error:{1}".format(keyname, e)) + self.logger.exception("Import Keypair {0} error:{1}".format(keyname, e)) return def get_server(self, openstack_id): @@ -339,10 +345,10 @@ def get_server(self, openstack_id): try: server = self.conn.compute.get_server(openstack_id) except Exception as e: - self.logger.error("No Server found {0} | Error {1}".format(openstack_id, e)) + self.logger.exception("No Server found {0} | Error {1}".format(openstack_id, e)) return VM(status="DELETED") if server is None: - self.logger.error("No Server {0}".format(openstack_id)) + self.logger.exception("No Server {0}".format(openstack_id)) raise serverNotFoundException(Reason="No Server {0}".format(openstack_id)) serv = server.to_dict() @@ -362,12 +368,12 @@ def get_server(self, openstack_id): try: flav = self.conn.compute.get_flavor(serv["flavor"]["id"]).to_dict() except Exception as e: - self.logger.error(e) + self.logger.exception(e) flav = None try: img = self.get_Image_with_Tag(serv["image"]["id"]) except Exception as e: - self.logger.error(e) + self.logger.exception(e) img = None for values in server.addresses.values(): for address in values: @@ -448,19 +454,19 @@ def start_server( metadata = {"elixir_id": elixir_id} image = self.conn.compute.find_image(image) if image is None: - self.logger.error("Image {0} not found!".format(image)) + self.logger.exception("Image {0} not found!".format(image)) raise imageNotFoundException( Reason=("Image {0} not fournd".format(image)) ) flavor = self.conn.compute.find_flavor(flavor) if flavor is None: - self.logger.error("Flavor {0} not found!".format(flavor)) + self.logger.exception("Flavor {0} not found!".format(flavor)) raise flavorNotFoundException( Reason="Flavor {0} not found!".format(flavor) ) network = self.conn.network.find_network(self.NETWORK) if network is None: - self.logger.error("Network {0} not found!".format(network)) + self.logger.exception("Network {0} not found!".format(network)) raise networkNotFoundException( Reason="Network {0} not found!".format(network) ) @@ -478,7 +484,7 @@ def start_server( name=volumename, size=int(diskspace) ).to_dict() except Exception as e: - self.logger.error( + self.logger.exception( "Trying to create volume with {0}" " GB for vm {1} error : {2}".format(diskspace, servername, e), exc_info=True, @@ -521,6 +527,76 @@ def start_server( self.logger.exception("Start Server {1} error:{0}".format(e, servername)) return {} + def create_and_deploy_playbook(self, private_key, play_source, openstack_id): + # create temporary directory and copy bioconda.yml and variables.yml there + #ancon_dir = "/code/VirtualMachineService/ancon" + #playbooks_dir = ancon_dir + "/playbooks" + #directory = TemporaryDirectory(dir=ancon_dir) + #shutil.copy(playbooks_dir+"/bioconda.yml", directory.name) + #shutil.copy(playbooks_dir+"/variables.yml", directory.name) + + # get ip and port for inventory + fields = self.get_ip_ports(openstack_id=openstack_id) + #ip = fields["IP"] + #port = fields["PORT"] + + playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source) + playbook.run_it() + + # FOR DEV PURPOSES ONLY! create private key +# private_key = NamedTemporaryFile(mode="w+", dir=directory.name, delete=False) +# private_key.write("""-----BEGIN RSA PRIVATE KEY----- +# MIIEpAIBAAKCAQEAvO3PMJojazQj1XljbTph404C74oQR3qRsWzhUyBERVlRsdJi +# Wv0i+LPOmFKDyZ5e7c5YHoPaqItgfc4L49f+v0TcwYyAJth85jv6VF4qz+hEzmi5 +# GlR3IXmUCk8JuyuU+o6oHqxr8RD3lBT76Lwu2FIN5OqXZlMMDwzkWex/8rrFVGyM +# JL1OSpnt6pspPdp7Nlxy7rakSprfalqhRlK0BWf/jkVx+VZwMQVZNNUlLaDIAzyO +# 03aEbnCAZVG+6OXbr9hZwwZ9WY+XfA013uGandntQQdShdop/nm6NirUyQE9gtzM +# omeoGt+u/YK1/2MbOouUlx2aEXkFWIJWrZd6owIDAQABAoIBAQCmP0jzRp9mJVJm +# 9dMk+ZvLjgkNSds7WsK7csjwAdOxhoBZznxX/qn4WRixduKa1u5HqixmZbZSW5sD +# +P0DeDyliG4NLppSFGwLmLmV5escWhG54/MGFU9jOH2peJVii14kAMY1f5nYXgrN +# 1o045eb+2W16g2fIVcmlsL17151bM7WD5p+Jsm6zxIR23pV+NAgY6rt/Kdy+y5O1 +# rh+oQrwEx0+k5ig6DIRkfgBvAKwreqaaPRHMryxOOR/CRxB6zLyoyatCEjCWDLcl +# cii0Zz95c6msHVBeZ4x2HeRulitLdvD9f4oYaojw277wg3mBDMWq2GZMc7xHl+6f +# 14cAsp0BAoGBAN2yXd9VH5ptiEwi8aKqc6RxPwiofCjnkfwAUiBeiHKbSMidnDDp +# +N4Azt1KFCIMWWTkxr/h1LHbhTBdJrLyR+HKHQ9qyJrqm3vNmk0WYW6PiX5LdlHL +# cKaRlyfjt7BGfjBetnrapcPuxQEQ1VIRZgXTaY5MIhrXgsUYH/GFyLSBAoGBANop +# egmuOHm0pk+DJyfivozhhYmWoNlA8x3aqdnZuseMonpLROBEtQrF0YsHHFHiRUvG +# K/6RYzVudpKSmB8grmmGmYNMwppD9ZJ55+/1cnFCQFmU1oENH1GDZhLmNTeinDXV +# LgFhCH+bR3Lp/6kdlEpXVXE9sinue4V/me3m3E0jAoGAKoj0VcshOyHUyrbRoaIO +# efh4XZLl73sumSj+mNNKXqLIfiUvOHtLklyZU//IiRfRdvgl4d7UTiOOFE9rA15U +# yE9c7/5O6tokZsZ12mB25R2JBcA4vUzJGkxIshCQx7Netq0VWdDliQggqCmwpARO +# jMOZNwIIcRn0LxiH2HEQpwECgYEAzzvDD1sNjp7JtJITKdI7y7uWjAInvPfzeRJz +# cdtfj5rJ5H2Haboad6c9y2Dvx+C2jqoqtGEK6oCJ5eWW10rGIruXK6BI4x1XMtLW +# PZzcHzYdxnqZ4HDEpTu6RI2lU7oFxSVB1FGGLyEjl8cr8kuEx7F6Gl3O1gISF9gE +# MnawIh0CgYBSmrR8kHDkHLWbPkKhH93qrJxbo0vSsk+F77Zb+0jA5yLk4SK8jhcx +# /p8iKdiMxnfR6BZyBUYMMW6AXx0Wg+iRQflka+/ugJUFyvELPm2s4ENRY68pTLj8 +# Zc+swBO6T87o29E6e4NLDtQ+h9MsoImfKOLRNKSZttOzfjW3MXqRMg== +# -----END RSA PRIVATE KEY-----""") +# private_key.close() +# +# # create inventory and add the to-be-installed tools to the variables.yml +# inventory = NamedTemporaryFile(mode="w+", dir=directory.name, delete=False) +# inventory_string = "[vm]\n" + ip + ":" + port + " ansible_user=ubuntu " \ +# "ansible_ssh_private_key_file=" + private_key.name +# inventory.write(inventory_string) +# inventory.close() +# yaml_exec = ruamel.yaml.YAML() +# #self.logger.info(play_source) +# #self.logger.info(play_source.strip('\"')) +# play_source = play_source.strip('\"') +# with open(directory.name + "/variables.yml", mode='r+') as variables: +# data = yaml_exec.load(variables) +# data["tools"]["string_line"] = play_source +# #self.logger.info(data["tools"]["string_line"]) +# yaml_exec.dump(data, variables) +# +# # start a subprocess which executes ansible-playbook with bioconda.yml and the inventory +# command_string = "/usr/local/bin/ansible-playbook -i " + inventory.name + " " + directory.name + "/bioconda.yml" +# command_string = shlex.split(command_string) +# process = subprocess.run(command_string, stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True) +# directory.cleanup() + return 0 + def create_volume(self, volume_name, diskspace): """ Create volume. @@ -537,7 +613,7 @@ def create_volume(self, volume_name, diskspace): volumeId = volume["id"] return volumeId except Exception as e: - self.logger.error( + self.logger.exception( "Trying to create volume with {0} GB error : {1}".format(diskspace, e), exc_info=True, ) @@ -572,7 +648,7 @@ def checkStatusVolume(volume, conn): server = self.conn.compute.get_server(openstack_id) if server is None: - self.logger.error("No Server {0} ".format(openstack_id)) + self.logger.exception("No Server {0} ".format(openstack_id)) raise serverNotFoundException(Reason="No Server {0}".format(openstack_id)) if checkStatusVolume(volume_id, self.conn): @@ -586,7 +662,7 @@ def checkStatusVolume(volume, conn): server=server, volumeId=volume_id ) except Exception as e: - self.logger.error( + self.logger.exception( "Trying to attache volume {0} to vm {1} error : {2}".format( volume_id, openstack_id, e ), @@ -615,10 +691,10 @@ def check_server_status(self, openstack_id, diskspace, volume_id): try: server = self.conn.compute.get_server(openstack_id) except Exception: - self.logger.error("No Server with id {0} ".format(openstack_id)) + self.logger.exception("No Server with id {0} ".format(openstack_id)) return None if server is None: - self.logger.error("No Server with id {0} ".format(openstack_id)) + self.logger.exception("No Server with id {0} ".format(openstack_id)) return None serv = server.to_dict() @@ -658,7 +734,7 @@ def check_server_status(self, openstack_id, diskspace, volume_id): server.status = "BUILD" return server except Exception as e: - self.logger.error("Check Status VM {0} error: {1}".format(openstack_id, e)) + self.logger.exception("Check Status VM {0} error: {1}".format(openstack_id, e)) return None def add_security_group_to_server(self, http, https, udp, server_id): @@ -739,7 +815,7 @@ def get_ip_ports(self, openstack_id): floating_ip = self.get_server(openstack_id) return {"IP": str(floating_ip)} except Exception as e: - self.logger.error( + self.logger.exception( "Get IP and PORT for server {0} error:".format(openstack_id, e) ) return {} @@ -765,7 +841,7 @@ def create_snapshot(self, openstack_id, name, elixir_id, base_tag): server=openstack_id, name=name ) except Exception: - self.logger.error("Instance {0} not found".format(openstack_id)) + self.logger.exception("Instance {0} not found".format(openstack_id)) return try: snapshot_id = snapshot_munch["id"] @@ -775,7 +851,7 @@ def create_snapshot(self, openstack_id, name, elixir_id, base_tag): ) return snapshot_id except Exception as e: - self.logger.error( + self.logger.exception( "Create Snapshot from Instance {0}" " with name {1} for {2} error : {3}".format( openstack_id, name, elixir_id, e @@ -794,14 +870,14 @@ def delete_image(self, image_id): try: image = self.conn.compute.get_image(image_id) if image is None: - self.logger.error("Image {0} not found!".format(image)) + self.logger.exception("Image {0} not found!".format(image)) raise imageNotFoundException( Reason=("Image {0} not found".format(image)) ) self.conn.compute.delete_image(image) return True except Exception as e: - self.logger.error("Delete Image {0} error : {1}".format(image_id, e)) + self.logger.exception("Delete Image {0} error : {1}".format(image_id, e)) return False def add_floating_ip_to_server(self, openstack_id, network): @@ -816,7 +892,7 @@ def add_floating_ip_to_server(self, openstack_id, network): server = self.conn.compute.get_server(openstack_id) if server is None: - self.logger.error("Instance {0} not found".format(openstack_id)) + self.logger.exception("Instance {0} not found".format(openstack_id)) raise serverNotFoundException self.logger.info("Checking if Server already got an Floating Ip") for values in server.addresses.values(): @@ -839,7 +915,7 @@ def add_floating_ip_to_server(self, openstack_id, network): networkID = self.conn.network.find_network(network) if networkID is None: - self.logger.error("Network " + network + " not found") + self.logger.exception("Network " + network + " not found") raise networkNotFoundException networkID = networkID.to_dict()["id"] floating_ip = self.conn.network.create_ip(floating_network_id=networkID) @@ -850,7 +926,7 @@ def add_floating_ip_to_server(self, openstack_id, network): return floating_ip except Exception as e: - self.logger.error( + self.logger.exception( "Adding Floating IP to {0} with network {1} error:{2}".format( openstack_id, network, e ) @@ -887,7 +963,7 @@ def delete_server(self, openstack_id): try: server = self.conn.compute.get_server(openstack_id) if server is None: - self.logger.error("Instance {0} not found".format(openstack_id)) + self.logger.exception("Instance {0} not found".format(openstack_id)) raise serverNotFoundException if server.status == "SUSPENDED": @@ -906,7 +982,7 @@ def delete_server(self, openstack_id): return True except Exception as e: - self.logger.error("Delete Server {0} error: {1}".format(openstack_id, e)) + self.logger.exception("Delete Server {0} error: {1}".format(openstack_id, e)) return False def delete_volume_attachment(self, volume_id, server_id): @@ -966,7 +1042,7 @@ def checkStatusVolume(volume, conn): self.conn.block_storage.delete_volume(volume=volume_id) return True except Exception as e: - self.logger.error("Delete Volume {0} error".format(volume_id, e)) + self.logger.exception("Delete Volume {0} error".format(volume_id, e)) return False def stop_server(self, openstack_id): @@ -980,7 +1056,7 @@ def stop_server(self, openstack_id): server = self.conn.compute.get_server(openstack_id) try: if server is None: - self.logger.error("Instance {0} not found".format(openstack_id)) + self.logger.exception("Instance {0} not found".format(openstack_id)) raise serverNotFoundException if server.status == "ACTIVE": @@ -995,7 +1071,7 @@ def stop_server(self, openstack_id): return False except Exception as e: - self.logger.error("Stop Server {0} error:".format(openstack_id, e)) + self.logger.exception("Stop Server {0} error:".format(openstack_id, e)) return False @@ -1011,7 +1087,7 @@ def reboot_server(self, server_id, reboot_type): try: server = self.conn.compute.get_server(server_id) if server is None: - self.logger.error("Instance {0} not found".format(server_id)) + self.logger.exception("Instance {0} not found".format(server_id)) raise serverNotFoundException else: self.conn.compute.reboot_server(server, reboot_type) @@ -1033,7 +1109,7 @@ def resume_server(self, openstack_id): try: server = self.conn.compute.get_server(openstack_id) if server is None: - self.logger.error("Instance {0} not found".format(openstack_id)) + self.logger.exception("Instance {0} not found".format(openstack_id)) raise serverNotFoundException if server.status == "SUSPENDED": @@ -1047,7 +1123,7 @@ def resume_server(self, openstack_id): return False except Exception as e: - self.logger.error("Resume Server {0} error:".format(openstack_id, e)) + self.logger.exception("Resume Server {0} error:".format(openstack_id, e)) return False def create_security_group( diff --git a/VirtualMachineService/VirtualMachineServer.py b/VirtualMachineService/VirtualMachineServer.py index e7cff268..0b6737c4 100644 --- a/VirtualMachineService/VirtualMachineServer.py +++ b/VirtualMachineService/VirtualMachineServer.py @@ -4,12 +4,12 @@ try: from VirtualMachineService import Client, Processor except Exception: - from .VirtualMachineService import Client, Processor + from VirtualMachineService import Client, Processor try: from VirtualMachineHandler import VirtualMachineHandler except Exception: - from .VirtualMachineHandler import VirtualMachineHandler + from VirtualMachineHandler import VirtualMachineHandler from thrift.transport import TSSLSocket from thrift.transport import TTransport diff --git a/VirtualMachineService/VirtualMachineService-remote b/VirtualMachineService/VirtualMachineService-remote index 1376ab87..7126b020 100755 --- a/VirtualMachineService/VirtualMachineService-remote +++ b/VirtualMachineService/VirtualMachineService-remote @@ -37,6 +37,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' string add_floating_ip_to_server(string openstack_id, string network)') print(' bool create_connection(string username, string password, string auth_url, string user_domain_name, string project_domain_name)') print(' start_server(string flavor, string image, string public_key, string servername, string elixir_id, string diskspace, string volumename)') + print(' int create_and_deploy_playbook(string private_key, string play_source, string openstack_id)') print(' bool add_security_group_to_server(bool http, bool https, bool udp, string server_id)') print(' VM get_server(string openstack_id)') print(' bool stop_server(string openstack_id)') @@ -208,6 +209,12 @@ elif cmd == 'start_server': sys.exit(1) pp.pprint(client.start_server(args[0], args[1], args[2], args[3], args[4], args[5], args[6],)) +elif cmd == 'create_and_deploy_playbook': + if len(args) != 3: + print('create_and_deploy_playbook requires 3 args') + sys.exit(1) + pp.pprint(client.create_and_deploy_playbook(args[0], args[1], args[2],)) + elif cmd == 'add_security_group_to_server': if len(args) != 4: print('add_security_group_to_server requires 4 args') diff --git a/VirtualMachineService/VirtualMachineService.py b/VirtualMachineService/VirtualMachineService.py index 58c6f81e..393d1be6 100644 --- a/VirtualMachineService/VirtualMachineService.py +++ b/VirtualMachineService/VirtualMachineService.py @@ -164,6 +164,18 @@ def start_server(self, flavor, image, public_key, servername, elixir_id, diskspa """ pass + def create_and_deploy_playbook(self, private_key, play_source, openstack_id): + """ + Create and deploy an anaconda ansible playbook + + Parameters: + - private_key + - play_source + - openstack_id + + """ + pass + def add_security_group_to_server(self, http, https, udp, server_id): """ Adds a security group to a server @@ -832,6 +844,44 @@ def recv_start_server(self): raise result.o raise TApplicationException(TApplicationException.MISSING_RESULT, "start_server failed: unknown result") + def create_and_deploy_playbook(self, private_key, play_source, openstack_id): + """ + Create and deploy an anaconda ansible playbook + + Parameters: + - private_key + - play_source + - openstack_id + + """ + self.send_create_and_deploy_playbook(private_key, play_source, openstack_id) + return self.recv_create_and_deploy_playbook() + + def send_create_and_deploy_playbook(self, private_key, play_source, openstack_id): + self._oprot.writeMessageBegin('create_and_deploy_playbook', TMessageType.CALL, self._seqid) + args = create_and_deploy_playbook_args() + args.private_key = private_key + args.play_source = play_source + args.openstack_id = openstack_id + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_and_deploy_playbook(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_and_deploy_playbook_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "create_and_deploy_playbook failed: unknown result") + def add_security_group_to_server(self, http, https, udp, server_id): """ Adds a security group to a server @@ -1392,6 +1442,7 @@ def __init__(self, handler): self._processMap["add_floating_ip_to_server"] = Processor.process_add_floating_ip_to_server self._processMap["create_connection"] = Processor.process_create_connection self._processMap["start_server"] = Processor.process_start_server + self._processMap["create_and_deploy_playbook"] = Processor.process_create_and_deploy_playbook self._processMap["add_security_group_to_server"] = Processor.process_add_security_group_to_server self._processMap["get_server"] = Processor.process_get_server self._processMap["stop_server"] = Processor.process_stop_server @@ -1760,6 +1811,29 @@ def process_start_server(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_create_and_deploy_playbook(self, seqid, iprot, oprot): + args = create_and_deploy_playbook_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_and_deploy_playbook_result() + try: + result.success = self._handler.create_and_deploy_playbook(args.private_key, args.play_source, args.openstack_id) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("create_and_deploy_playbook", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_add_security_group_to_server(self, seqid, iprot, oprot): args = add_security_group_to_server_args() args.read(iprot) @@ -4085,6 +4159,153 @@ def __ne__(self, other): ) +class create_and_deploy_playbook_args(object): + """ + Attributes: + - private_key + - play_source + - openstack_id + + """ + + + def __init__(self, private_key=None, play_source=None, openstack_id=None,): + self.private_key = private_key + self.play_source = play_source + self.openstack_id = openstack_id + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.private_key = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.play_source = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.openstack_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('create_and_deploy_playbook_args') + if self.private_key is not None: + oprot.writeFieldBegin('private_key', TType.STRING, 1) + oprot.writeString(self.private_key.encode('utf-8') if sys.version_info[0] == 2 else self.private_key) + oprot.writeFieldEnd() + if self.play_source is not None: + oprot.writeFieldBegin('play_source', TType.STRING, 2) + oprot.writeString(self.play_source.encode('utf-8') if sys.version_info[0] == 2 else self.play_source) + oprot.writeFieldEnd() + if self.openstack_id is not None: + oprot.writeFieldBegin('openstack_id', TType.STRING, 3) + oprot.writeString(self.openstack_id.encode('utf-8') if sys.version_info[0] == 2 else self.openstack_id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(create_and_deploy_playbook_args) +create_and_deploy_playbook_args.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'private_key', 'UTF8', None, ), # 1 + (2, TType.STRING, 'play_source', 'UTF8', None, ), # 2 + (3, TType.STRING, 'openstack_id', 'UTF8', None, ), # 3 +) + + +class create_and_deploy_playbook_result(object): + """ + Attributes: + - success + + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('create_and_deploy_playbook_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(create_and_deploy_playbook_result) +create_and_deploy_playbook_result.thrift_spec = ( + (0, TType.I32, 'success', None, None, ), # 0 +) + + class add_security_group_to_server_args(object): """ Attributes: diff --git a/VirtualMachineService/ancon/BiocondaPlaybook.py b/VirtualMachineService/ancon/BiocondaPlaybook.py new file mode 100644 index 00000000..501f02e2 --- /dev/null +++ b/VirtualMachineService/ancon/BiocondaPlaybook.py @@ -0,0 +1,66 @@ +import shlex +import shutil +import sys +from tempfile import NamedTemporaryFile, TemporaryDirectory +import ruamel.yaml +import subprocess + + +class BiocondaPlaybook(object): + + def __init__(self, ip, port, play_source): + self.ancon_dir = "/code/VirtualMachineService/ancon" + self.playbooks_dir = self.ancon_dir + "/playbooks" + self.directory = TemporaryDirectory(dir=self.ancon_dir) + shutil.copy(self.playbooks_dir+"/bioconda.yml", self.directory.name) + shutil.copy(self.playbooks_dir+"/variables.yml", self.directory.name) + self.ip = ip + self.port = port + self.private_key = NamedTemporaryFile(mode="w+", dir=self.directory.name, delete=False) + self.private_key.write("""-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAvO3PMJojazQj1XljbTph404C74oQR3qRsWzhUyBERVlRsdJi +Wv0i+LPOmFKDyZ5e7c5YHoPaqItgfc4L49f+v0TcwYyAJth85jv6VF4qz+hEzmi5 +GlR3IXmUCk8JuyuU+o6oHqxr8RD3lBT76Lwu2FIN5OqXZlMMDwzkWex/8rrFVGyM +JL1OSpnt6pspPdp7Nlxy7rakSprfalqhRlK0BWf/jkVx+VZwMQVZNNUlLaDIAzyO +03aEbnCAZVG+6OXbr9hZwwZ9WY+XfA013uGandntQQdShdop/nm6NirUyQE9gtzM +omeoGt+u/YK1/2MbOouUlx2aEXkFWIJWrZd6owIDAQABAoIBAQCmP0jzRp9mJVJm +9dMk+ZvLjgkNSds7WsK7csjwAdOxhoBZznxX/qn4WRixduKa1u5HqixmZbZSW5sD ++P0DeDyliG4NLppSFGwLmLmV5escWhG54/MGFU9jOH2peJVii14kAMY1f5nYXgrN +1o045eb+2W16g2fIVcmlsL17151bM7WD5p+Jsm6zxIR23pV+NAgY6rt/Kdy+y5O1 +rh+oQrwEx0+k5ig6DIRkfgBvAKwreqaaPRHMryxOOR/CRxB6zLyoyatCEjCWDLcl +cii0Zz95c6msHVBeZ4x2HeRulitLdvD9f4oYaojw277wg3mBDMWq2GZMc7xHl+6f +14cAsp0BAoGBAN2yXd9VH5ptiEwi8aKqc6RxPwiofCjnkfwAUiBeiHKbSMidnDDp ++N4Azt1KFCIMWWTkxr/h1LHbhTBdJrLyR+HKHQ9qyJrqm3vNmk0WYW6PiX5LdlHL +cKaRlyfjt7BGfjBetnrapcPuxQEQ1VIRZgXTaY5MIhrXgsUYH/GFyLSBAoGBANop +egmuOHm0pk+DJyfivozhhYmWoNlA8x3aqdnZuseMonpLROBEtQrF0YsHHFHiRUvG +K/6RYzVudpKSmB8grmmGmYNMwppD9ZJ55+/1cnFCQFmU1oENH1GDZhLmNTeinDXV +LgFhCH+bR3Lp/6kdlEpXVXE9sinue4V/me3m3E0jAoGAKoj0VcshOyHUyrbRoaIO +efh4XZLl73sumSj+mNNKXqLIfiUvOHtLklyZU//IiRfRdvgl4d7UTiOOFE9rA15U +yE9c7/5O6tokZsZ12mB25R2JBcA4vUzJGkxIshCQx7Netq0VWdDliQggqCmwpARO +jMOZNwIIcRn0LxiH2HEQpwECgYEAzzvDD1sNjp7JtJITKdI7y7uWjAInvPfzeRJz +cdtfj5rJ5H2Haboad6c9y2Dvx+C2jqoqtGEK6oCJ5eWW10rGIruXK6BI4x1XMtLW +PZzcHzYdxnqZ4HDEpTu6RI2lU7oFxSVB1FGGLyEjl8cr8kuEx7F6Gl3O1gISF9gE +MnawIh0CgYBSmrR8kHDkHLWbPkKhH93qrJxbo0vSsk+F77Zb+0jA5yLk4SK8jhcx +/p8iKdiMxnfR6BZyBUYMMW6AXx0Wg+iRQflka+/ugJUFyvELPm2s4ENRY68pTLj8 +Zc+swBO6T87o29E6e4NLDtQ+h9MsoImfKOLRNKSZttOzfjW3MXqRMg== +-----END RSA PRIVATE KEY-----""") + self.private_key.close() + + # create inventory and add the to-be-installed tools to the variables.yml + self.inventory = NamedTemporaryFile(mode="w+", dir=self.directory.name, delete=False) + inventory_string = "[vm]\n" + self.ip + ":" + self.port + " ansible_user=ubuntu " \ + "ansible_ssh_private_key_file=" + self.private_key.name + self.inventory.write(inventory_string) + self.inventory.close() + yaml_exec = ruamel.yaml.YAML() + self.play_source = play_source.strip('\"') + with open(self.directory.name + "/variables.yml", mode='r+') as variables: + data = yaml_exec.load(variables) + data["tools"]["string_line"] = play_source + yaml_exec.dump(data, variables) + + def run_it(self): + command_string = "/usr/local/bin/ansible-playbook -i " + self.inventory.name + " " + self.directory.name + "/bioconda.yml" + command_string = shlex.split(command_string) + process = subprocess.run(command_string, stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True) + self.directory.cleanup() diff --git a/VirtualMachineService/ancon/callback.py b/VirtualMachineService/ancon/callback.py new file mode 100644 index 00000000..87ac283a --- /dev/null +++ b/VirtualMachineService/ancon/callback.py @@ -0,0 +1,199 @@ +from datetime import datetime +from ansible.plugins.callback import CallbackBase +import logging +import os +#from django.conf import settings + + +class PlayLogger: + """Store log output in a single object. + We create a new object per Ansible run + """ + def __init__(self): + self.log = '' + self.runtime = 0 + + def append(self, log_line): + """append to log""" + self.log += log_line+"\n\n" + + def banner(self, msg): + """Output Trailing Stars""" + width = 78 - len(msg) + if width < 3: + width = 3 + filler = "*" * width + return "\n%s %s " % (msg, filler) + + +class CallbackModule(CallbackBase): + """ + Reference: https://github.com/ansible/ansible/blob/v2.0.0.2-1/lib/ansible/plugins/callback/default.py + """ + + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'stored' + CALLBACK_NAME = 'database' + + def __init__(self): + super(CallbackModule, self).__init__() + self.logger = PlayLogger() + self.start_time = datetime.now() + + def v2_runner_on_failed(self, result, ignore_errors=False): + delegated_vars = result._result.get('_ansible_delegated_vars', None) + + # Catch an exception + # This may never be called because default handler deletes + # the exception, since Ansible thinks it knows better + if 'exception' in result._result: + # Extract the error message and log it + error = result._result['exception'].strip().split('\n')[-1] + self.logger.append(error) + + # Remove the exception from the result so it's not shown every time + del result._result['exception'] + + # Else log the reason for the failure + if result._task.loop and 'results' in result._result: + self._process_items(result) # item_on_failed, item_on_skipped, item_on_ok + else: + if delegated_vars: + self.logger.append("fatal: [%s -> %s]: FAILED! => %s" % (result._host.get_name(), + delegated_vars['ansible_host'], + self._dump_results(result._result))) + else: + self.logger.append("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result))) + + def v2_runner_on_ok(self, result): + self._clean_results(result._result, result._task.action) + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if result._task.action == 'include': + return + elif result._result.get('changed', False): + if delegated_vars: + msg = "changed: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) + else: + msg = "changed: [%s]" % result._host.get_name() + else: + if delegated_vars: + msg = "ok: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) + else: + msg = "ok: [%s]" % result._host.get_name() + + if result._task.loop and 'results' in result._result: + self._process_items(result) # item_on_failed, item_on_skipped, item_on_ok + else: + self.logger.append(msg) + + def v2_runner_on_skipped(self, result): + if result._task.loop and 'results' in result._result: + self._process_items(result) # item_on_failed, item_on_skipped, item_on_ok + else: + msg = "skipping: [%s]" % result._host.get_name() + self.logger.append(msg) + + def v2_runner_on_unreachable(self, result): + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if delegated_vars: + self.logger.append("fatal: [%s -> %s]: UNREACHABLE! => %s" % (result._host.get_name(), + delegated_vars['ansible_host'], + self._dump_results(result._result))) + else: + self.logger.append("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), + self._dump_results(result._result))) + + def v2_runner_on_no_hosts(self, task): + self.logger.append("skipping: no hosts matched") + + def v2_playbook_on_task_start(self, task, is_conditional): + self.logger.append("TASK [%s]" % task.get_name().strip()) + + def v2_playbook_on_play_start(self, play): + name = play.get_name().strip() + if not name: + msg = "PLAY" + else: + msg = "PLAY [%s]" % name + + self.logger.append(msg) + + def v2_playbook_item_on_ok(self, result): + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if result._task.action == 'include': + return + elif result._result.get('changed', False): + if delegated_vars: + msg = "changed: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) + else: + msg = "changed: [%s]" % result._host.get_name() + else: + if delegated_vars: + msg = "ok: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) + else: + msg = "ok: [%s]" % result._host.get_name() + + msg += " => (item=%s)" % (result._result['item']) + + self.logger.append(msg) + + def v2_playbook_item_on_failed(self, result): + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if 'exception' in result._result: + # Extract the error message and log it + error = result._result['exception'].strip().split('\n')[-1] + self.logger.append(error) + + # Remove the exception from the result so it's not shown every time + del result._result['exception'] + + if delegated_vars: + self.logger.append("failed: [%s -> %s] => (item=%s) => %s" % (result._host.get_name(), + delegated_vars['ansible_host'], + result._result['item'], + self._dump_results(result._result))) + else: + self.logger.append("failed: [%s] => (item=%s) => %s" % (result._host.get_name(), + result._result['item'], + self._dump_results(result._result))) + + def v2_playbook_item_on_skipped(self, result): + msg = "skipping: [%s] => (item=%s) " % (result._host.get_name(), result._result['item']) + self.logger.append(msg) + + def v2_playbook_on_stats(self, stats): + self.logger.runtimee = datetime.now() - self.start_time + #self.logger.runtime = run_time.seconds # returns an int, unlike run_time.total_seconds() + + hosts = sorted(stats.processed.keys()) + for h in hosts: + t = stats.summarize(h) + + msg = "PLAY RECAP [%s] : %s %s %s %s %s" % ( + h, + "ok: %s" % (t['ok']), + "changed: %s" % (t['changed']), + "unreachable: %s" % (t['unreachable']), + "skipped: %s" % (t['skipped']), + "failed: %s" % (t['failures']), + ) + + self.logger.append(msg) + + def record_logs(self): + """ + Special callback added to this callback plugin + Called by Runner objet + :param user_id: + :return: + """ + + logger = logging.getLogger(__name__) + fh = logging.FileHandler('log/ansible_playbook.log', delay=True) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(message)s') + fh.setFormatter(formatter) + logger.addHandler(fh) + print(self.start_time) + print(datetime.now() - self.start_time) + return logger.log(level=1000, msg=self.logger.log + str(self.logger.runtime)) diff --git a/VirtualMachineService/ancon/playbooks/bioconda.yml b/VirtualMachineService/ancon/playbooks/bioconda.yml new file mode 100644 index 00000000..a7e3746e --- /dev/null +++ b/VirtualMachineService/ancon/playbooks/bioconda.yml @@ -0,0 +1,35 @@ +- name: Install miniconda3 and a Test-Package + hosts: all + connection: paramiko_ssh + gather_facts: yes + vars_files: + - variables.yml + tasks: + + - name: Download miniconda install script + get_url: + args: + url: "{{ folders.conda_installer_url }}" + dest: "{{ folders.install_script }}" + mode: 0755 + + - name: Install miniconda + shell: "{{ folders.install_script }} -b" + args: + executable: /bin/bash + creates: "{{ folders.conda_dir }}" + + - name: Add Channels + shell: "source {{ folders.conda_dir }}/bin/activate && conda config --add channels default && conda config --add channels bioconda && conda config --add channels conda-forge" + args: + executable: /bin/bash + + - name: Create Environment + shell: "source {{ folders.conda_dir }}/bin/activate && conda create --yes -n {{ tools.env }}" + args: + executable: /bin/bash + + - name: Install Test-Package + shell: "source {{ folders.conda_dir }}/bin/activate && conda activate {{ tools.env }} && conda install --yes {{ tools.string_line }}" + args: + executable: /bin/bash diff --git a/VirtualMachineService/ancon/playbooks/variables.yml b/VirtualMachineService/ancon/playbooks/variables.yml new file mode 100644 index 00000000..cf9546ee --- /dev/null +++ b/VirtualMachineService/ancon/playbooks/variables.yml @@ -0,0 +1,8 @@ +tools: + string_line: "" + env: "denbi" + +folders: + install_script: "/home/{{ ansible_user_id }}/install_miniconda3.sh" + conda_dir: "/home/{{ ansible_user_id }}/miniconda3" + conda_installer_url: "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 00000000..5412b882 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,7 @@ +[defaults] +host_key_checking = False + +[paramiko_connection] +host_key_checking = False +env:ANSIBLE_PARAMIKO_HOST_KEY_CHECKING +var: ansible_paramiko_host_key_checking diff --git a/portal_client.thrift b/portal_client.thrift index 11d01bf2..062e5ffd 100644 --- a/portal_client.thrift +++ b/portal_client.thrift @@ -299,8 +299,14 @@ service VirtualMachineService { /** Name of additional Volume*/ 7:string volumename) - throws (1:nameException e,2:ressourceException r,3:serverNotFoundException s,4: networkNotFoundException n,5:imageNotFoundException i,6:flavorNotFoundException f,7:otherException o), - + throws (1:nameException e,2:ressourceException r,3:serverNotFoundException s,4: networkNotFoundException n,5:imageNotFoundException i,6:flavorNotFoundException f,7:otherException o) + + /** Create and deploy an anaconda ansible playbook*/ + int create_and_deploy_playbook( + 1:string private_key, + 2:string play_source, + 3:string openstack_id + ) /** * Adds a security group to a server diff --git a/requirements.txt b/requirements.txt index ecaa22d8..7f988207 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,11 @@ setuptools==40.4.3 -thrift >= 0.10.0,<0.20.0 +thrift >= 0.11.0,<0.20.0 python-keystoneclient openstacksdk ==0.27.0 deprecated == 1.2.4 Click==7.0 -flake8 +flake8==3.7.7 +ansible==2.8.0 +ruamel.yaml<0.16.00 +paramiko==2.6.0 +pyvim==2.0.24 From a555a5feb289e34946c25f8bf816d47dfab933ae Mon Sep 17 00:00:00 2001 From: David Weinholz Date: Thu, 4 Jul 2019 14:13:26 +0200 Subject: [PATCH 03/18] added custom ke ymetho --- .../VirtualMachineHandler.py | 221 ++++++--- .../VirtualMachineService-remote | 7 + .../VirtualMachineService.py | 418 +++++++++++++++++- portal_client.thrift | 26 ++ 4 files changed, 599 insertions(+), 73 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 7d470426..afaf719a 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -3,13 +3,6 @@ Which can be used for the PortalClient. """ -from tempfile import NamedTemporaryFile, TemporaryDirectory -import shutil -import ruamel.yaml -import subprocess -import shlex -import sys -from VirtualMachineService.ancon.BiocondaPlaybook import BiocondaPlaybook try: from VirtualMachineService import Iface @@ -22,6 +15,8 @@ from ttypes import ressourceException from ttypes import Flavor, Image, VM from constants import VERSION + from ancon.BiocondaPlaybook import BiocondaPlaybook + except Exception: from .VirtualMachineService import Iface from .ttypes import serverNotFoundException @@ -33,6 +28,8 @@ from .ttypes import ressourceException from .ttypes import Flavor, Image, VM from .constants import VERSION + from .ancon.BiocondaPlaybook import BiocondaPlaybook + from openstack import connection from deprecated import deprecated from keystoneauth1.identity import v3 @@ -49,6 +46,7 @@ import base64 from oslo_utils import encodeutils + class VirtualMachineHandler(Iface): """Handler which the PortalClient uses.""" @@ -527,74 +525,167 @@ def start_server( self.logger.exception("Start Server {1} error:{0}".format(e, servername)) return {} + def start_server_with_custom_key(self, flavor, image, servername, elixir_id, diskspace, + volumename): + + """ + Start a new Server. + + :param flavor: Name of flavor which should be used. + :param image: Name of image which should be used + :param public_key: Publickey which should be used + :param servername: Name of the new server + :param elixir_id: Elixir_id of the user who started a new server + :param diskspace: Diskspace in GB for volume which should be created + :param volumename: Name of the volume + :return: {'openstackid': serverId, 'volumeId': volumeId} + """ + volumeId = "" + self.logger.info("Start Server {} with custom key".format(servername)) + try: + metadata = {"elixir_id": elixir_id} + image = self.conn.compute.find_image(image) + if image is None: + self.logger.exception("Image {0} not found!".format(image)) + raise imageNotFoundException( + Reason=("Image {0} not fournd".format(image)) + ) + flavor = self.conn.compute.find_flavor(flavor) + if flavor is None: + self.logger.exception("Flavor {0} not found!".format(flavor)) + raise flavorNotFoundException( + Reason="Flavor {0} not found!".format(flavor) + ) + network = self.conn.network.find_network(self.NETWORK) + if network is None: + self.logger.exception("Network {0} not found!".format(network)) + raise networkNotFoundException( + Reason="Network {0} not found!".format(network) + ) + + private_key = self.conn.create_keypair(name=servername).__dict__['private_key'] + self.logger.info("Private key {}".format(private_key)) + + if diskspace > "0": + self.logger.info( + "Creating volume with {0} GB diskspace".format(diskspace) + ) + try: + volume = self.conn.block_storage.create_volume( + name=volumename, size=int(diskspace) + ).to_dict() + except Exception as e: + self.logger.exception( + "Trying to create volume with {0}" + " GB for vm {1} error : {2}".format(diskspace, servername, e), + exc_info=True, + ) + raise ressourceException(Reason=str(e)) + volumeId = volume["id"] + + fileDir = os.path.dirname(os.path.abspath(__file__)) + mount_script = os.path.join(fileDir, "scripts/bash/mount.sh") + with open(mount_script, "r") as file: + text = file.read() + text = text.replace("VOLUMEID", "virtio-" + volumeId[0:20]) + text = encodeutils.safe_encode(text.encode("utf-8")) + init_script = base64.b64encode(text).decode("utf-8") + + server = self.conn.compute.create_server( + name=servername, + image_id=image.id, + flavor_id=flavor.id, + networks=[{"uuid": network.id}], + key_name=servername, + metadata=metadata, + user_data=init_script, + availability_zone=self.AVAIALABILITY_ZONE, + ) + else: + server = self.conn.compute.create_server( + name=servername, + image_id=image.id, + flavor_id=flavor.id, + networks=[{"uuid": network.id}], + key_name=servername, + metadata=metadata, + ) + + openstack_id = server.to_dict()["id"] + + return {"openstackid": openstack_id, "volumeId": volumeId, 'private_key': private_key} + except Exception as e: + self.logger.exception("Start Server {1} error:{0}".format(e, servername)) + return {} + def create_and_deploy_playbook(self, private_key, play_source, openstack_id): # create temporary directory and copy bioconda.yml and variables.yml there - #ancon_dir = "/code/VirtualMachineService/ancon" - #playbooks_dir = ancon_dir + "/playbooks" - #directory = TemporaryDirectory(dir=ancon_dir) - #shutil.copy(playbooks_dir+"/bioconda.yml", directory.name) - #shutil.copy(playbooks_dir+"/variables.yml", directory.name) + # ancon_dir = "/code/VirtualMachineService/ancon" + # playbooks_dir = ancon_dir + "/playbooks" + # directory = TemporaryDirectory(dir=ancon_dir) + # shutil.copy(playbooks_dir+"/bioconda.yml", directory.name) + # shutil.copy(playbooks_dir+"/variables.yml", directory.name) # get ip and port for inventory fields = self.get_ip_ports(openstack_id=openstack_id) - #ip = fields["IP"] - #port = fields["PORT"] + # ip = fields["IP"] + # port = fields["PORT"] playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source) playbook.run_it() # FOR DEV PURPOSES ONLY! create private key -# private_key = NamedTemporaryFile(mode="w+", dir=directory.name, delete=False) -# private_key.write("""-----BEGIN RSA PRIVATE KEY----- -# MIIEpAIBAAKCAQEAvO3PMJojazQj1XljbTph404C74oQR3qRsWzhUyBERVlRsdJi -# Wv0i+LPOmFKDyZ5e7c5YHoPaqItgfc4L49f+v0TcwYyAJth85jv6VF4qz+hEzmi5 -# GlR3IXmUCk8JuyuU+o6oHqxr8RD3lBT76Lwu2FIN5OqXZlMMDwzkWex/8rrFVGyM -# JL1OSpnt6pspPdp7Nlxy7rakSprfalqhRlK0BWf/jkVx+VZwMQVZNNUlLaDIAzyO -# 03aEbnCAZVG+6OXbr9hZwwZ9WY+XfA013uGandntQQdShdop/nm6NirUyQE9gtzM -# omeoGt+u/YK1/2MbOouUlx2aEXkFWIJWrZd6owIDAQABAoIBAQCmP0jzRp9mJVJm -# 9dMk+ZvLjgkNSds7WsK7csjwAdOxhoBZznxX/qn4WRixduKa1u5HqixmZbZSW5sD -# +P0DeDyliG4NLppSFGwLmLmV5escWhG54/MGFU9jOH2peJVii14kAMY1f5nYXgrN -# 1o045eb+2W16g2fIVcmlsL17151bM7WD5p+Jsm6zxIR23pV+NAgY6rt/Kdy+y5O1 -# rh+oQrwEx0+k5ig6DIRkfgBvAKwreqaaPRHMryxOOR/CRxB6zLyoyatCEjCWDLcl -# cii0Zz95c6msHVBeZ4x2HeRulitLdvD9f4oYaojw277wg3mBDMWq2GZMc7xHl+6f -# 14cAsp0BAoGBAN2yXd9VH5ptiEwi8aKqc6RxPwiofCjnkfwAUiBeiHKbSMidnDDp -# +N4Azt1KFCIMWWTkxr/h1LHbhTBdJrLyR+HKHQ9qyJrqm3vNmk0WYW6PiX5LdlHL -# cKaRlyfjt7BGfjBetnrapcPuxQEQ1VIRZgXTaY5MIhrXgsUYH/GFyLSBAoGBANop -# egmuOHm0pk+DJyfivozhhYmWoNlA8x3aqdnZuseMonpLROBEtQrF0YsHHFHiRUvG -# K/6RYzVudpKSmB8grmmGmYNMwppD9ZJ55+/1cnFCQFmU1oENH1GDZhLmNTeinDXV -# LgFhCH+bR3Lp/6kdlEpXVXE9sinue4V/me3m3E0jAoGAKoj0VcshOyHUyrbRoaIO -# efh4XZLl73sumSj+mNNKXqLIfiUvOHtLklyZU//IiRfRdvgl4d7UTiOOFE9rA15U -# yE9c7/5O6tokZsZ12mB25R2JBcA4vUzJGkxIshCQx7Netq0VWdDliQggqCmwpARO -# jMOZNwIIcRn0LxiH2HEQpwECgYEAzzvDD1sNjp7JtJITKdI7y7uWjAInvPfzeRJz -# cdtfj5rJ5H2Haboad6c9y2Dvx+C2jqoqtGEK6oCJ5eWW10rGIruXK6BI4x1XMtLW -# PZzcHzYdxnqZ4HDEpTu6RI2lU7oFxSVB1FGGLyEjl8cr8kuEx7F6Gl3O1gISF9gE -# MnawIh0CgYBSmrR8kHDkHLWbPkKhH93qrJxbo0vSsk+F77Zb+0jA5yLk4SK8jhcx -# /p8iKdiMxnfR6BZyBUYMMW6AXx0Wg+iRQflka+/ugJUFyvELPm2s4ENRY68pTLj8 -# Zc+swBO6T87o29E6e4NLDtQ+h9MsoImfKOLRNKSZttOzfjW3MXqRMg== -# -----END RSA PRIVATE KEY-----""") -# private_key.close() -# -# # create inventory and add the to-be-installed tools to the variables.yml -# inventory = NamedTemporaryFile(mode="w+", dir=directory.name, delete=False) -# inventory_string = "[vm]\n" + ip + ":" + port + " ansible_user=ubuntu " \ -# "ansible_ssh_private_key_file=" + private_key.name -# inventory.write(inventory_string) -# inventory.close() -# yaml_exec = ruamel.yaml.YAML() -# #self.logger.info(play_source) -# #self.logger.info(play_source.strip('\"')) -# play_source = play_source.strip('\"') -# with open(directory.name + "/variables.yml", mode='r+') as variables: -# data = yaml_exec.load(variables) -# data["tools"]["string_line"] = play_source -# #self.logger.info(data["tools"]["string_line"]) -# yaml_exec.dump(data, variables) -# -# # start a subprocess which executes ansible-playbook with bioconda.yml and the inventory -# command_string = "/usr/local/bin/ansible-playbook -i " + inventory.name + " " + directory.name + "/bioconda.yml" -# command_string = shlex.split(command_string) -# process = subprocess.run(command_string, stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True) -# directory.cleanup() + # private_key = NamedTemporaryFile(mode="w+", dir=directory.name, delete=False) + # private_key.write("""-----BEGIN RSA PRIVATE KEY----- + # MIIEpAIBAAKCAQEAvO3PMJojazQj1XljbTph404C74oQR3qRsWzhUyBERVlRsdJi + # Wv0i+LPOmFKDyZ5e7c5YHoPaqItgfc4L49f+v0TcwYyAJth85jv6VF4qz+hEzmi5 + # GlR3IXmUCk8JuyuU+o6oHqxr8RD3lBT76Lwu2FIN5OqXZlMMDwzkWex/8rrFVGyM + # JL1OSpnt6pspPdp7Nlxy7rakSprfalqhRlK0BWf/jkVx+VZwMQVZNNUlLaDIAzyO + # 03aEbnCAZVG+6OXbr9hZwwZ9WY+XfA013uGandntQQdShdop/nm6NirUyQE9gtzM + # omeoGt+u/YK1/2MbOouUlx2aEXkFWIJWrZd6owIDAQABAoIBAQCmP0jzRp9mJVJm + # 9dMk+ZvLjgkNSds7WsK7csjwAdOxhoBZznxX/qn4WRixduKa1u5HqixmZbZSW5sD + # +P0DeDyliG4NLppSFGwLmLmV5escWhG54/MGFU9jOH2peJVii14kAMY1f5nYXgrN + # 1o045eb+2W16g2fIVcmlsL17151bM7WD5p+Jsm6zxIR23pV+NAgY6rt/Kdy+y5O1 + # rh+oQrwEx0+k5ig6DIRkfgBvAKwreqaaPRHMryxOOR/CRxB6zLyoyatCEjCWDLcl + # cii0Zz95c6msHVBeZ4x2HeRulitLdvD9f4oYaojw277wg3mBDMWq2GZMc7xHl+6f + # 14cAsp0BAoGBAN2yXd9VH5ptiEwi8aKqc6RxPwiofCjnkfwAUiBeiHKbSMidnDDp + # +N4Azt1KFCIMWWTkxr/h1LHbhTBdJrLyR+HKHQ9qyJrqm3vNmk0WYW6PiX5LdlHL + # cKaRlyfjt7BGfjBetnrapcPuxQEQ1VIRZgXTaY5MIhrXgsUYH/GFyLSBAoGBANop + # egmuOHm0pk+DJyfivozhhYmWoNlA8x3aqdnZuseMonpLROBEtQrF0YsHHFHiRUvG + # K/6RYzVudpKSmB8grmmGmYNMwppD9ZJ55+/1cnFCQFmU1oENH1GDZhLmNTeinDXV + # LgFhCH+bR3Lp/6kdlEpXVXE9sinue4V/me3m3E0jAoGAKoj0VcshOyHUyrbRoaIO + # efh4XZLl73sumSj+mNNKXqLIfiUvOHtLklyZU//IiRfRdvgl4d7UTiOOFE9rA15U + # yE9c7/5O6tokZsZ12mB25R2JBcA4vUzJGkxIshCQx7Netq0VWdDliQggqCmwpARO + # jMOZNwIIcRn0LxiH2HEQpwECgYEAzzvDD1sNjp7JtJITKdI7y7uWjAInvPfzeRJz + # cdtfj5rJ5H2Haboad6c9y2Dvx+C2jqoqtGEK6oCJ5eWW10rGIruXK6BI4x1XMtLW + # PZzcHzYdxnqZ4HDEpTu6RI2lU7oFxSVB1FGGLyEjl8cr8kuEx7F6Gl3O1gISF9gE + # MnawIh0CgYBSmrR8kHDkHLWbPkKhH93qrJxbo0vSsk+F77Zb+0jA5yLk4SK8jhcx + # /p8iKdiMxnfR6BZyBUYMMW6AXx0Wg+iRQflka+/ugJUFyvELPm2s4ENRY68pTLj8 + # Zc+swBO6T87o29E6e4NLDtQ+h9MsoImfKOLRNKSZttOzfjW3MXqRMg== + # -----END RSA PRIVATE KEY-----""") + # private_key.close() + # + # # create inventory and add the to-be-installed tools to the variables.yml + # inventory = NamedTemporaryFile(mode="w+", dir=directory.name, delete=False) + # inventory_string = "[vm]\n" + ip + ":" + port + " ansible_user=ubuntu " \ + # "ansible_ssh_private_key_file=" + private_key.name + # inventory.write(inventory_string) + # inventory.close() + # yaml_exec = ruamel.yaml.YAML() + # #self.logger.info(play_source) + # #self.logger.info(play_source.strip('\"')) + # play_source = play_source.strip('\"') + # with open(directory.name + "/variables.yml", mode='r+') as variables: + # data = yaml_exec.load(variables) + # data["tools"]["string_line"] = play_source + # #self.logger.info(data["tools"]["string_line"]) + # yaml_exec.dump(data, variables) + # + # # start a subprocess which executes ansible-playbook with bioconda.yml and the inventory + # command_string = "/usr/local/bin/ansible-playbook -i " + inventory.name + " " + directory.name + "/bioconda.yml" + # command_string = shlex.split(command_string) + # process = subprocess.run(command_string, stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True) + # directory.cleanup() return 0 def create_volume(self, volume_name, diskspace): diff --git a/VirtualMachineService/VirtualMachineService-remote b/VirtualMachineService/VirtualMachineService-remote index 1579abf2..25a95af0 100755 --- a/VirtualMachineService/VirtualMachineService-remote +++ b/VirtualMachineService/VirtualMachineService-remote @@ -37,6 +37,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' string add_floating_ip_to_server(string openstack_id, string network)') print(' bool create_connection(string username, string password, string auth_url, string user_domain_name, string project_domain_name)') print(' start_server(string flavor, string image, string public_key, string servername, string elixir_id, string diskspace, string volumename)') + print(' start_server_with_custom_key(string flavor, string image, string servername, string elixir_id, string diskspace, string volumename)') print(' int create_and_deploy_playbook(string private_key, string play_source, string openstack_id)') print(' bool add_security_group_to_server(bool http, bool https, bool udp, string server_id)') print(' VM get_server(string openstack_id)') @@ -209,6 +210,12 @@ elif cmd == 'start_server': sys.exit(1) pp.pprint(client.start_server(args[0], args[1], args[2], args[3], args[4], args[5], args[6],)) +elif cmd == 'start_server_with_custom_key': + if len(args) != 6: + print('start_server_with_custom_key requires 6 args') + sys.exit(1) + pp.pprint(client.start_server_with_custom_key(args[0], args[1], args[2], args[3], args[4], args[5],)) + elif cmd == 'create_and_deploy_playbook': if len(args) != 3: print('create_and_deploy_playbook requires 3 args') diff --git a/VirtualMachineService/VirtualMachineService.py b/VirtualMachineService/VirtualMachineService.py index 6a2160b9..9aa7fd20 100644 --- a/VirtualMachineService/VirtualMachineService.py +++ b/VirtualMachineService/VirtualMachineService.py @@ -164,6 +164,21 @@ def start_server(self, flavor, image, public_key, servername, elixir_id, diskspa """ pass + def start_server_with_custom_key(self, flavor, image, servername, elixir_id, diskspace, volumename): + """ + Start a new server. + + Parameters: + - flavor: Name of the Flavor to use. + - image: Name of the image to use. + - servername: Name for the new server + - elixir_id: Elixir-Id of the user who requested to start a new server + - diskspace: Diskspace in GB for additional volume. + - volumename: Name of additional Volume + + """ + pass + def create_and_deploy_playbook(self, private_key, play_source, openstack_id): """ Create and deploy an anaconda ansible playbook @@ -845,6 +860,64 @@ def recv_start_server(self): raise result.o raise TApplicationException(TApplicationException.MISSING_RESULT, "start_server failed: unknown result") + def start_server_with_custom_key(self, flavor, image, servername, elixir_id, diskspace, volumename): + """ + Start a new server. + + Parameters: + - flavor: Name of the Flavor to use. + - image: Name of the image to use. + - servername: Name for the new server + - elixir_id: Elixir-Id of the user who requested to start a new server + - diskspace: Diskspace in GB for additional volume. + - volumename: Name of additional Volume + + """ + self.send_start_server_with_custom_key(flavor, image, servername, elixir_id, diskspace, volumename) + return self.recv_start_server_with_custom_key() + + def send_start_server_with_custom_key(self, flavor, image, servername, elixir_id, diskspace, volumename): + self._oprot.writeMessageBegin('start_server_with_custom_key', TMessageType.CALL, self._seqid) + args = start_server_with_custom_key_args() + args.flavor = flavor + args.image = image + args.servername = servername + args.elixir_id = elixir_id + args.diskspace = diskspace + args.volumename = volumename + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_start_server_with_custom_key(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = start_server_with_custom_key_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.e is not None: + raise result.e + if result.r is not None: + raise result.r + if result.s is not None: + raise result.s + if result.n is not None: + raise result.n + if result.i is not None: + raise result.i + if result.f is not None: + raise result.f + if result.o is not None: + raise result.o + raise TApplicationException(TApplicationException.MISSING_RESULT, "start_server_with_custom_key failed: unknown result") + def create_and_deploy_playbook(self, private_key, play_source, openstack_id): """ Create and deploy an anaconda ansible playbook @@ -1445,6 +1518,7 @@ def __init__(self, handler): self._processMap["add_floating_ip_to_server"] = Processor.process_add_floating_ip_to_server self._processMap["create_connection"] = Processor.process_create_connection self._processMap["start_server"] = Processor.process_start_server + self._processMap["start_server_with_custom_key"] = Processor.process_start_server_with_custom_key self._processMap["create_and_deploy_playbook"] = Processor.process_create_and_deploy_playbook self._processMap["add_security_group_to_server"] = Processor.process_add_security_group_to_server self._processMap["get_server"] = Processor.process_get_server @@ -1814,6 +1888,50 @@ def process_start_server(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_start_server_with_custom_key(self, seqid, iprot, oprot): + args = start_server_with_custom_key_args() + args.read(iprot) + iprot.readMessageEnd() + result = start_server_with_custom_key_result() + try: + result.success = self._handler.start_server_with_custom_key(args.flavor, args.image, args.servername, args.elixir_id, args.diskspace, args.volumename) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except nameException as e: + msg_type = TMessageType.REPLY + result.e = e + except ressourceException as r: + msg_type = TMessageType.REPLY + result.r = r + except serverNotFoundException as s: + msg_type = TMessageType.REPLY + result.s = s + except networkNotFoundException as n: + msg_type = TMessageType.REPLY + result.n = n + except imageNotFoundException as i: + msg_type = TMessageType.REPLY + result.i = i + except flavorNotFoundException as f: + msg_type = TMessageType.REPLY + result.f = f + except otherException as o: + msg_type = TMessageType.REPLY + result.o = o + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("start_server_with_custom_key", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_create_and_deploy_playbook(self, seqid, iprot, oprot): args = create_and_deploy_playbook_args() args.read(iprot) @@ -4162,6 +4280,290 @@ def __ne__(self, other): ) +class start_server_with_custom_key_args(object): + """ + Attributes: + - flavor: Name of the Flavor to use. + - image: Name of the image to use. + - servername: Name for the new server + - elixir_id: Elixir-Id of the user who requested to start a new server + - diskspace: Diskspace in GB for additional volume. + - volumename: Name of additional Volume + + """ + + + def __init__(self, flavor=None, image=None, servername=None, elixir_id=None, diskspace=None, volumename=None,): + self.flavor = flavor + self.image = image + self.servername = servername + self.elixir_id = elixir_id + self.diskspace = diskspace + self.volumename = volumename + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.flavor = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.image = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.servername = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.elixir_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.diskspace = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.volumename = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('start_server_with_custom_key_args') + if self.flavor is not None: + oprot.writeFieldBegin('flavor', TType.STRING, 1) + oprot.writeString(self.flavor.encode('utf-8') if sys.version_info[0] == 2 else self.flavor) + oprot.writeFieldEnd() + if self.image is not None: + oprot.writeFieldBegin('image', TType.STRING, 2) + oprot.writeString(self.image.encode('utf-8') if sys.version_info[0] == 2 else self.image) + oprot.writeFieldEnd() + if self.servername is not None: + oprot.writeFieldBegin('servername', TType.STRING, 3) + oprot.writeString(self.servername.encode('utf-8') if sys.version_info[0] == 2 else self.servername) + oprot.writeFieldEnd() + if self.elixir_id is not None: + oprot.writeFieldBegin('elixir_id', TType.STRING, 4) + oprot.writeString(self.elixir_id.encode('utf-8') if sys.version_info[0] == 2 else self.elixir_id) + oprot.writeFieldEnd() + if self.diskspace is not None: + oprot.writeFieldBegin('diskspace', TType.STRING, 5) + oprot.writeString(self.diskspace.encode('utf-8') if sys.version_info[0] == 2 else self.diskspace) + oprot.writeFieldEnd() + if self.volumename is not None: + oprot.writeFieldBegin('volumename', TType.STRING, 6) + oprot.writeString(self.volumename.encode('utf-8') if sys.version_info[0] == 2 else self.volumename) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(start_server_with_custom_key_args) +start_server_with_custom_key_args.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'flavor', 'UTF8', None, ), # 1 + (2, TType.STRING, 'image', 'UTF8', None, ), # 2 + (3, TType.STRING, 'servername', 'UTF8', None, ), # 3 + (4, TType.STRING, 'elixir_id', 'UTF8', None, ), # 4 + (5, TType.STRING, 'diskspace', 'UTF8', None, ), # 5 + (6, TType.STRING, 'volumename', 'UTF8', None, ), # 6 +) + + +class start_server_with_custom_key_result(object): + """ + Attributes: + - success + - e + - r + - s + - n + - i + - f + - o + + """ + + + def __init__(self, success=None, e=None, r=None, s=None, n=None, i=None, f=None, o=None,): + self.success = success + self.e = e + self.r = r + self.s = s + self.n = n + self.i = i + self.f = f + self.o = o + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.MAP: + self.success = {} + (_ktype88, _vtype89, _size87) = iprot.readMapBegin() + for _i91 in range(_size87): + _key92 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + _val93 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + self.success[_key92] = _val93 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.e = nameException() + self.e.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.r = ressourceException() + self.r.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.s = serverNotFoundException() + self.s.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.n = networkNotFoundException() + self.n.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.i = imageNotFoundException() + self.i.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRUCT: + self.f = flavorNotFoundException() + self.f.read(iprot) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRUCT: + self.o = otherException() + self.o.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('start_server_with_custom_key_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.MAP, 0) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.success)) + for kiter94, viter95 in self.success.items(): + oprot.writeString(kiter94.encode('utf-8') if sys.version_info[0] == 2 else kiter94) + oprot.writeString(viter95.encode('utf-8') if sys.version_info[0] == 2 else viter95) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.e is not None: + oprot.writeFieldBegin('e', TType.STRUCT, 1) + self.e.write(oprot) + oprot.writeFieldEnd() + if self.r is not None: + oprot.writeFieldBegin('r', TType.STRUCT, 2) + self.r.write(oprot) + oprot.writeFieldEnd() + if self.s is not None: + oprot.writeFieldBegin('s', TType.STRUCT, 3) + self.s.write(oprot) + oprot.writeFieldEnd() + if self.n is not None: + oprot.writeFieldBegin('n', TType.STRUCT, 4) + self.n.write(oprot) + oprot.writeFieldEnd() + if self.i is not None: + oprot.writeFieldBegin('i', TType.STRUCT, 5) + self.i.write(oprot) + oprot.writeFieldEnd() + if self.f is not None: + oprot.writeFieldBegin('f', TType.STRUCT, 6) + self.f.write(oprot) + oprot.writeFieldEnd() + if self.o is not None: + oprot.writeFieldBegin('o', TType.STRUCT, 7) + self.o.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(start_server_with_custom_key_result) +start_server_with_custom_key_result.thrift_spec = ( + (0, TType.MAP, 'success', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 0 + (1, TType.STRUCT, 'e', [nameException, None], None, ), # 1 + (2, TType.STRUCT, 'r', [ressourceException, None], None, ), # 2 + (3, TType.STRUCT, 's', [serverNotFoundException, None], None, ), # 3 + (4, TType.STRUCT, 'n', [networkNotFoundException, None], None, ), # 4 + (5, TType.STRUCT, 'i', [imageNotFoundException, None], None, ), # 5 + (6, TType.STRUCT, 'f', [flavorNotFoundException, None], None, ), # 6 + (7, TType.STRUCT, 'o', [otherException, None], None, ), # 7 +) + + class create_and_deploy_playbook_args(object): """ Attributes: @@ -5017,11 +5419,11 @@ def read(self, iprot): if fid == 0: if ftype == TType.MAP: self.success = {} - (_ktype88, _vtype89, _size87) = iprot.readMapBegin() - for _i91 in range(_size87): - _key92 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - _val93 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.success[_key92] = _val93 + (_ktype97, _vtype98, _size96) = iprot.readMapBegin() + for _i100 in range(_size96): + _key101 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + _val102 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + self.success[_key101] = _val102 iprot.readMapEnd() else: iprot.skip(ftype) @@ -5038,9 +5440,9 @@ def write(self, oprot): if self.success is not None: oprot.writeFieldBegin('success', TType.MAP, 0) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.success)) - for kiter94, viter95 in self.success.items(): - oprot.writeString(kiter94.encode('utf-8') if sys.version_info[0] == 2 else kiter94) - oprot.writeString(viter95.encode('utf-8') if sys.version_info[0] == 2 else viter95) + for kiter103, viter104 in self.success.items(): + oprot.writeString(kiter103.encode('utf-8') if sys.version_info[0] == 2 else kiter103) + oprot.writeString(viter104.encode('utf-8') if sys.version_info[0] == 2 else viter104) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() diff --git a/portal_client.thrift b/portal_client.thrift index 9522336d..d1682f85 100644 --- a/portal_client.thrift +++ b/portal_client.thrift @@ -301,6 +301,32 @@ service VirtualMachineService { throws (1:nameException e,2:ressourceException r,3:serverNotFoundException s,4: networkNotFoundException n,5:imageNotFoundException i,6:flavorNotFoundException f,7:otherException o) + + /** + * Start a new server. + */ + map start_server_with_custom_key( + + /** Name of the Flavor to use.*/ + 1:string flavor, + + /** Name of the image to use. */ + 2:string image, + + /** Name for the new server */ + 3:string servername, + + /** Elixir-Id of the user who requested to start a new server*/ + 4:string elixir_id, + + /** Diskspace in GB for additional volume.*/ + 5:string diskspace, + + /** Name of additional Volume*/ + 6:string volumename) + + throws (1:nameException e,2:ressourceException r,3:serverNotFoundException s,4: networkNotFoundException n,5:imageNotFoundException i,6:flavorNotFoundException f,7:otherException o) + /** Create and deploy an anaconda ansible playbook*/ int create_and_deploy_playbook( 1:string private_key, From 31a96b5d5f2919406763ab954ed46cdc24e4c104 Mon Sep 17 00:00:00 2001 From: David Weinholz Date: Thu, 4 Jul 2019 14:41:05 +0200 Subject: [PATCH 04/18] refactored a little bit --- .../VirtualMachineHandler.py | 170 ++++++++---------- 1 file changed, 75 insertions(+), 95 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index afaf719a..02f3fe66 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -424,6 +424,60 @@ def get_server(self, openstack_id): ) return server + def get_image(self, image): + image = self.conn.compute.find_image(image) + if image is None: + self.logger.exception("Image {0} not found!".format(image)) + raise imageNotFoundException( + Reason=("Image {0} not found".format(image)) + ) + return image + + def get_flavor(self, flavor): + flavor = self.conn.compute.find_flavor(flavor) + if flavor is None: + self.logger.exception("Flavor {0} not found!".format(flavor)) + raise flavorNotFoundException( + Reason="Flavor {0} not found!".format(flavor) + ) + return flavor + + def get_network(self): + network = self.conn.network.find_network(self.NETWORK) + if network is None: + self.logger.exception("Network {0} not found!".format(network)) + raise networkNotFoundException( + Reason="Network {0} not found!".format(network) + ) + return network + + def create_volume(self, volume_storage, volume_name, server_name): + self.logger.info( + "Creating volume with {0} GB diskspace".format(volume_storage) + ) + try: + volume = self.conn.block_storage.create_volume( + name=volume_name, size=int(volume_storage) + ).to_dict() + except Exception as e: + self.logger.exception( + "Trying to create volume with {0}" + " GB for vm {1} error : {2}".format(volume_storage, server_name, e), + exc_info=True, + ) + raise ressourceException(Reason=str(e)) + return volume["id"] + + def create_mount_init_script(self, volume_id): + fileDir = os.path.dirname(os.path.abspath(__file__)) + mount_script = os.path.join(fileDir, "scripts/bash/mount.sh") + with open(mount_script, "r") as file: + text = file.read() + text = text.replace("VOLUMEID", "virtio-" + volume_id[0:20]) + text = encodeutils.safe_encode(text.encode("utf-8")) + init_script = base64.b64encode(text).decode("utf-8") + return init_script + def start_server( self, flavor, @@ -446,64 +500,28 @@ def start_server( :param volumename: Name of the volume :return: {'openstackid': serverId, 'volumeId': volumeId} """ - volumeId = "" + volume_id = '' self.logger.info("Start Server {0}".format(servername)) try: metadata = {"elixir_id": elixir_id} - image = self.conn.compute.find_image(image) - if image is None: - self.logger.exception("Image {0} not found!".format(image)) - raise imageNotFoundException( - Reason=("Image {0} not fournd".format(image)) - ) - flavor = self.conn.compute.find_flavor(flavor) - if flavor is None: - self.logger.exception("Flavor {0} not found!".format(flavor)) - raise flavorNotFoundException( - Reason="Flavor {0} not found!".format(flavor) - ) - network = self.conn.network.find_network(self.NETWORK) - if network is None: - self.logger.exception("Network {0} not found!".format(network)) - raise networkNotFoundException( - Reason="Network {0} not found!".format(network) - ) - - keyname = elixir_id[:-18] + image = self.get_image(image=image) + flavor = self.get_flavor(flavor=flavor) + network = self.get_network() + key_name = elixir_id[:-18] public_key = urllib.parse.unquote(public_key) - keypair = self.import_keypair(keyname, public_key) + key_pair = self.import_keypair(key_name, public_key) if diskspace > "0": - self.logger.info( - "Creating volume with {0} GB diskspace".format(diskspace) - ) - try: - volume = self.conn.block_storage.create_volume( - name=volumename, size=int(diskspace) - ).to_dict() - except Exception as e: - self.logger.exception( - "Trying to create volume with {0}" - " GB for vm {1} error : {2}".format(diskspace, servername, e), - exc_info=True, - ) - raise ressourceException(Reason=str(e)) - volumeId = volume["id"] - - fileDir = os.path.dirname(os.path.abspath(__file__)) - mount_script = os.path.join(fileDir, "scripts/bash/mount.sh") - with open(mount_script, "r") as file: - text = file.read() - text = text.replace("VOLUMEID", "virtio-" + volumeId[0:20]) - text = encodeutils.safe_encode(text.encode("utf-8")) - init_script = base64.b64encode(text).decode("utf-8") + volume_id = self.create_volume(volume_storage=diskspace, volume_name=volumename, + server_name=servername) + init_script = self.create_mount_init_script(volume_id=volume_id) server = self.conn.compute.create_server( name=servername, image_id=image.id, flavor_id=flavor.id, networks=[{"uuid": network.id}], - key_name=keypair.name, + key_name=key_pair.name, metadata=metadata, user_data=init_script, availability_zone=self.AVAIALABILITY_ZONE, @@ -514,13 +532,13 @@ def start_server( image_id=image.id, flavor_id=flavor.id, networks=[{"uuid": network.id}], - key_name=keypair.name, + key_name=key_pair.name, metadata=metadata, ) openstack_id = server.to_dict()["id"] - return {"openstackid": openstack_id, "volumeId": volumeId} + return {"openstackid": openstack_id, "volumeId": volume_id} except Exception as e: self.logger.exception("Start Server {1} error:{0}".format(e, servername)) return {} @@ -540,56 +558,18 @@ def start_server_with_custom_key(self, flavor, image, servername, elixir_id, dis :param volumename: Name of the volume :return: {'openstackid': serverId, 'volumeId': volumeId} """ - volumeId = "" self.logger.info("Start Server {} with custom key".format(servername)) + volume_id = '' try: metadata = {"elixir_id": elixir_id} - image = self.conn.compute.find_image(image) - if image is None: - self.logger.exception("Image {0} not found!".format(image)) - raise imageNotFoundException( - Reason=("Image {0} not fournd".format(image)) - ) - flavor = self.conn.compute.find_flavor(flavor) - if flavor is None: - self.logger.exception("Flavor {0} not found!".format(flavor)) - raise flavorNotFoundException( - Reason="Flavor {0} not found!".format(flavor) - ) - network = self.conn.network.find_network(self.NETWORK) - if network is None: - self.logger.exception("Network {0} not found!".format(network)) - raise networkNotFoundException( - Reason="Network {0} not found!".format(network) - ) - + image = self.get_image(image=image) + flavor = self.get_flavor(flavor=flavor) + network = self.get_network() private_key = self.conn.create_keypair(name=servername).__dict__['private_key'] - self.logger.info("Private key {}".format(private_key)) - - if diskspace > "0": - self.logger.info( - "Creating volume with {0} GB diskspace".format(diskspace) - ) - try: - volume = self.conn.block_storage.create_volume( - name=volumename, size=int(diskspace) - ).to_dict() - except Exception as e: - self.logger.exception( - "Trying to create volume with {0}" - " GB for vm {1} error : {2}".format(diskspace, servername, e), - exc_info=True, - ) - raise ressourceException(Reason=str(e)) - volumeId = volume["id"] - - fileDir = os.path.dirname(os.path.abspath(__file__)) - mount_script = os.path.join(fileDir, "scripts/bash/mount.sh") - with open(mount_script, "r") as file: - text = file.read() - text = text.replace("VOLUMEID", "virtio-" + volumeId[0:20]) - text = encodeutils.safe_encode(text.encode("utf-8")) - init_script = base64.b64encode(text).decode("utf-8") + if int(diskspace) > 0: + volume_id = self.create_volume(volume_storage=diskspace, volume_name=volumename, + server_name=servername) + init_script = self.create_mount_init_script(volume_id=volume_id) server = self.conn.compute.create_server( name=servername, @@ -613,7 +593,7 @@ def start_server_with_custom_key(self, flavor, image, servername, elixir_id, dis openstack_id = server.to_dict()["id"] - return {"openstackid": openstack_id, "volumeId": volumeId, 'private_key': private_key} + return {"openstackid": openstack_id, "volumeId": volume_id, 'private_key': private_key} except Exception as e: self.logger.exception("Start Server {1} error:{0}".format(e, servername)) return {} From 00b5e29959291da9e225d45dc464e515c080ee7a Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 14:52:03 +0200 Subject: [PATCH 05/18] global dict --- VirtualMachineService/VirtualMachineHandler.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 02f3fe66..35bc3fab 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -46,10 +46,14 @@ import base64 from oslo_utils import encodeutils +osi_key_dict = dict() + class VirtualMachineHandler(Iface): """Handler which the PortalClient uses.""" + global osi_key_dict + def create_connection(self): """ Create connection to OpenStack. @@ -592,7 +596,8 @@ def start_server_with_custom_key(self, flavor, image, servername, elixir_id, dis ) openstack_id = server.to_dict()["id"] - + global osi_key_dict + osi_key_dict[openstack_id] = private_key return {"openstackid": openstack_id, "volumeId": volume_id, 'private_key': private_key} except Exception as e: self.logger.exception("Start Server {1} error:{0}".format(e, servername)) @@ -610,7 +615,8 @@ def create_and_deploy_playbook(self, private_key, play_source, openstack_id): fields = self.get_ip_ports(openstack_id=openstack_id) # ip = fields["IP"] # port = fields["PORT"] - + global osi_key_dict + self.logger.info(osi_key_dict) playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source) playbook.run_it() From b8e28764c27a7023ac56b0b38c8ab436b8e36000 Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 14:58:33 +0200 Subject: [PATCH 06/18] start ansible with created private key --- .../VirtualMachineHandler.py | 3 +- .../ancon/BiocondaPlaybook.py | 30 ++----------------- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 35bc3fab..0c2c03a0 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -616,8 +616,7 @@ def create_and_deploy_playbook(self, private_key, play_source, openstack_id): # ip = fields["IP"] # port = fields["PORT"] global osi_key_dict - self.logger.info(osi_key_dict) - playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source) + playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id]) playbook.run_it() # FOR DEV PURPOSES ONLY! create private key diff --git a/VirtualMachineService/ancon/BiocondaPlaybook.py b/VirtualMachineService/ancon/BiocondaPlaybook.py index 501f02e2..87260c87 100644 --- a/VirtualMachineService/ancon/BiocondaPlaybook.py +++ b/VirtualMachineService/ancon/BiocondaPlaybook.py @@ -8,7 +8,7 @@ class BiocondaPlaybook(object): - def __init__(self, ip, port, play_source): + def __init__(self, ip, port, play_source, osi_private_key): self.ancon_dir = "/code/VirtualMachineService/ancon" self.playbooks_dir = self.ancon_dir + "/playbooks" self.directory = TemporaryDirectory(dir=self.ancon_dir) @@ -17,33 +17,7 @@ def __init__(self, ip, port, play_source): self.ip = ip self.port = port self.private_key = NamedTemporaryFile(mode="w+", dir=self.directory.name, delete=False) - self.private_key.write("""-----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAvO3PMJojazQj1XljbTph404C74oQR3qRsWzhUyBERVlRsdJi -Wv0i+LPOmFKDyZ5e7c5YHoPaqItgfc4L49f+v0TcwYyAJth85jv6VF4qz+hEzmi5 -GlR3IXmUCk8JuyuU+o6oHqxr8RD3lBT76Lwu2FIN5OqXZlMMDwzkWex/8rrFVGyM -JL1OSpnt6pspPdp7Nlxy7rakSprfalqhRlK0BWf/jkVx+VZwMQVZNNUlLaDIAzyO -03aEbnCAZVG+6OXbr9hZwwZ9WY+XfA013uGandntQQdShdop/nm6NirUyQE9gtzM -omeoGt+u/YK1/2MbOouUlx2aEXkFWIJWrZd6owIDAQABAoIBAQCmP0jzRp9mJVJm -9dMk+ZvLjgkNSds7WsK7csjwAdOxhoBZznxX/qn4WRixduKa1u5HqixmZbZSW5sD -+P0DeDyliG4NLppSFGwLmLmV5escWhG54/MGFU9jOH2peJVii14kAMY1f5nYXgrN -1o045eb+2W16g2fIVcmlsL17151bM7WD5p+Jsm6zxIR23pV+NAgY6rt/Kdy+y5O1 -rh+oQrwEx0+k5ig6DIRkfgBvAKwreqaaPRHMryxOOR/CRxB6zLyoyatCEjCWDLcl -cii0Zz95c6msHVBeZ4x2HeRulitLdvD9f4oYaojw277wg3mBDMWq2GZMc7xHl+6f -14cAsp0BAoGBAN2yXd9VH5ptiEwi8aKqc6RxPwiofCjnkfwAUiBeiHKbSMidnDDp -+N4Azt1KFCIMWWTkxr/h1LHbhTBdJrLyR+HKHQ9qyJrqm3vNmk0WYW6PiX5LdlHL -cKaRlyfjt7BGfjBetnrapcPuxQEQ1VIRZgXTaY5MIhrXgsUYH/GFyLSBAoGBANop -egmuOHm0pk+DJyfivozhhYmWoNlA8x3aqdnZuseMonpLROBEtQrF0YsHHFHiRUvG -K/6RYzVudpKSmB8grmmGmYNMwppD9ZJ55+/1cnFCQFmU1oENH1GDZhLmNTeinDXV -LgFhCH+bR3Lp/6kdlEpXVXE9sinue4V/me3m3E0jAoGAKoj0VcshOyHUyrbRoaIO -efh4XZLl73sumSj+mNNKXqLIfiUvOHtLklyZU//IiRfRdvgl4d7UTiOOFE9rA15U -yE9c7/5O6tokZsZ12mB25R2JBcA4vUzJGkxIshCQx7Netq0VWdDliQggqCmwpARO -jMOZNwIIcRn0LxiH2HEQpwECgYEAzzvDD1sNjp7JtJITKdI7y7uWjAInvPfzeRJz -cdtfj5rJ5H2Haboad6c9y2Dvx+C2jqoqtGEK6oCJ5eWW10rGIruXK6BI4x1XMtLW -PZzcHzYdxnqZ4HDEpTu6RI2lU7oFxSVB1FGGLyEjl8cr8kuEx7F6Gl3O1gISF9gE -MnawIh0CgYBSmrR8kHDkHLWbPkKhH93qrJxbo0vSsk+F77Zb+0jA5yLk4SK8jhcx -/p8iKdiMxnfR6BZyBUYMMW6AXx0Wg+iRQflka+/ugJUFyvELPm2s4ENRY68pTLj8 -Zc+swBO6T87o29E6e4NLDtQ+h9MsoImfKOLRNKSZttOzfjW3MXqRMg== ------END RSA PRIVATE KEY-----""") + self.private_key.write(osi_private_key) self.private_key.close() # create inventory and add the to-be-installed tools to the variables.yml From dafd80455eca7614353430986ceeb8360806f60a Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 15:05:28 +0200 Subject: [PATCH 07/18] fix(ansible): fix string.strip error --- VirtualMachineService/ancon/BiocondaPlaybook.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/VirtualMachineService/ancon/BiocondaPlaybook.py b/VirtualMachineService/ancon/BiocondaPlaybook.py index 87260c87..898ee57f 100644 --- a/VirtualMachineService/ancon/BiocondaPlaybook.py +++ b/VirtualMachineService/ancon/BiocondaPlaybook.py @@ -27,10 +27,9 @@ def __init__(self, ip, port, play_source, osi_private_key): self.inventory.write(inventory_string) self.inventory.close() yaml_exec = ruamel.yaml.YAML() - self.play_source = play_source.strip('\"') with open(self.directory.name + "/variables.yml", mode='r+') as variables: data = yaml_exec.load(variables) - data["tools"]["string_line"] = play_source + data["tools"]["string_line"] = play_source.strip('\"') yaml_exec.dump(data, variables) def run_it(self): From aef0200e223936be6cf14b97052d81f62683716d Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 15:23:26 +0200 Subject: [PATCH 08/18] feature(ansible): exchange public keys --- VirtualMachineService/VirtualMachineHandler.py | 2 +- VirtualMachineService/ancon/BiocondaPlaybook.py | 3 ++- VirtualMachineService/ancon/playbooks/bioconda.yml | 7 +++++++ VirtualMachineService/ancon/playbooks/variables.yml | 1 + 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 0c2c03a0..916d88e2 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -616,7 +616,7 @@ def create_and_deploy_playbook(self, private_key, play_source, openstack_id): # ip = fields["IP"] # port = fields["PORT"] global osi_key_dict - playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id]) + playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id], private_key) playbook.run_it() # FOR DEV PURPOSES ONLY! create private key diff --git a/VirtualMachineService/ancon/BiocondaPlaybook.py b/VirtualMachineService/ancon/BiocondaPlaybook.py index 898ee57f..8e27526d 100644 --- a/VirtualMachineService/ancon/BiocondaPlaybook.py +++ b/VirtualMachineService/ancon/BiocondaPlaybook.py @@ -8,7 +8,7 @@ class BiocondaPlaybook(object): - def __init__(self, ip, port, play_source, osi_private_key): + def __init__(self, ip, port, play_source, osi_private_key, public_key): self.ancon_dir = "/code/VirtualMachineService/ancon" self.playbooks_dir = self.ancon_dir + "/playbooks" self.directory = TemporaryDirectory(dir=self.ancon_dir) @@ -30,6 +30,7 @@ def __init__(self, ip, port, play_source, osi_private_key): with open(self.directory.name + "/variables.yml", mode='r+') as variables: data = yaml_exec.load(variables) data["tools"]["string_line"] = play_source.strip('\"') + data["tools"]["public_key"] = public_key.strip('\"') yaml_exec.dump(data, variables) def run_it(self): diff --git a/VirtualMachineService/ancon/playbooks/bioconda.yml b/VirtualMachineService/ancon/playbooks/bioconda.yml index a7e3746e..55fc325d 100644 --- a/VirtualMachineService/ancon/playbooks/bioconda.yml +++ b/VirtualMachineService/ancon/playbooks/bioconda.yml @@ -33,3 +33,10 @@ shell: "source {{ folders.conda_dir }}/bin/activate && conda activate {{ tools.env }} && conda install --yes {{ tools.string_line }}" args: executable: /bin/bash + + - name: Set Public Key and remove old Public Key + authorized_key: + user: ubuntu + key: '{{ tools.public_key }}' + state: present + exclusive: True diff --git a/VirtualMachineService/ancon/playbooks/variables.yml b/VirtualMachineService/ancon/playbooks/variables.yml index cf9546ee..58eb3fc8 100644 --- a/VirtualMachineService/ancon/playbooks/variables.yml +++ b/VirtualMachineService/ancon/playbooks/variables.yml @@ -1,6 +1,7 @@ tools: string_line: "" env: "denbi" + public_key: "" folders: install_script: "/home/{{ ansible_user_id }}/install_miniconda3.sh" From 131118f994baf03fff62907e540612c949710000 Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 15:35:19 +0200 Subject: [PATCH 09/18] save key name --- VirtualMachineService/VirtualMachineHandler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 916d88e2..04436094 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -597,7 +597,7 @@ def start_server_with_custom_key(self, flavor, image, servername, elixir_id, dis openstack_id = server.to_dict()["id"] global osi_key_dict - osi_key_dict[openstack_id] = private_key + osi_key_dict[openstack_id] = dict(key=private_key, name=servername) return {"openstackid": openstack_id, "volumeId": volume_id, 'private_key': private_key} except Exception as e: self.logger.exception("Start Server {1} error:{0}".format(e, servername)) @@ -616,7 +616,7 @@ def create_and_deploy_playbook(self, private_key, play_source, openstack_id): # ip = fields["IP"] # port = fields["PORT"] global osi_key_dict - playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id], private_key) + playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id]["key"], private_key) playbook.run_it() # FOR DEV PURPOSES ONLY! create private key From e8d016cfcb7a0f1d2aace1b404bdf8be7e534407 Mon Sep 17 00:00:00 2001 From: David Weinholz Date: Thu, 4 Jul 2019 16:01:02 +0200 Subject: [PATCH 10/18] delete custom key --- .../VirtualMachineHandler.py | 46 ++++++++----------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 04436094..17d906be 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -305,6 +305,11 @@ def get_Image_with_Tag(self, id): self.logger.exception("Get Image {0} with Tag Error: {1}".format(id, e)) return + def delete_keypair(self, key_name): + key_pair = self.conn.compute.find_keypair(key_name) + if key_pair: + self.conn.compute.delete_keypair(key_pair) + def import_keypair(self, keyname, public_key): """ Import Keypair to OpenStack. @@ -455,7 +460,7 @@ def get_network(self): ) return network - def create_volume(self, volume_storage, volume_name, server_name): + def create_volume_by_start(self, volume_storage, volume_name, server_name): self.logger.info( "Creating volume with {0} GB diskspace".format(volume_storage) ) @@ -516,8 +521,9 @@ def start_server( key_pair = self.import_keypair(key_name, public_key) if diskspace > "0": - volume_id = self.create_volume(volume_storage=diskspace, volume_name=volumename, - server_name=servername) + volume_id = self.create_volume_by_start(volume_storage=diskspace, + volume_name=volumename, + server_name=servername) init_script = self.create_mount_init_script(volume_id=volume_id) server = self.conn.compute.create_server( @@ -571,8 +577,9 @@ def start_server_with_custom_key(self, flavor, image, servername, elixir_id, dis network = self.get_network() private_key = self.conn.create_keypair(name=servername).__dict__['private_key'] if int(diskspace) > 0: - volume_id = self.create_volume(volume_storage=diskspace, volume_name=volumename, - server_name=servername) + volume_id = self.create_volume_by_start(volume_storage=diskspace, + volume_name=volumename, + server_name=servername) init_script = self.create_mount_init_script(volume_id=volume_id) server = self.conn.compute.create_server( @@ -600,6 +607,7 @@ def start_server_with_custom_key(self, flavor, image, servername, elixir_id, dis osi_key_dict[openstack_id] = dict(key=private_key, name=servername) return {"openstackid": openstack_id, "volumeId": volume_id, 'private_key': private_key} except Exception as e: + self.delete_keypair(key_name=servername) self.logger.exception("Start Server {1} error:{0}".format(e, servername)) return {} @@ -616,7 +624,9 @@ def create_and_deploy_playbook(self, private_key, play_source, openstack_id): # ip = fields["IP"] # port = fields["PORT"] global osi_key_dict - playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id]["key"], private_key) + key_name = osi_key_dict[openstack_id]['name'] + playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, + osi_key_dict[openstack_id]["key"], private_key) playbook.run_it() # FOR DEV PURPOSES ONLY! create private key @@ -671,29 +681,9 @@ def create_and_deploy_playbook(self, private_key, play_source, openstack_id): # command_string = shlex.split(command_string) # process = subprocess.run(command_string, stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True) # directory.cleanup() - return 0 - - def create_volume(self, volume_name, diskspace): - """ - Create volume. - :param volume_name: Name of volume - :param diskspace: Diskspace in GB for new volume - :return: Id of new volume - """ - self.logger.info("Creating volume with {0} GB diskspace".format(diskspace)) - try: - volume = self.conn.block_storage.create_volume( - name=volume_name, size=int(diskspace) - ).to_dict() - volumeId = volume["id"] - return volumeId - except Exception as e: - self.logger.exception( - "Trying to create volume with {0} GB error : {1}".format(diskspace, e), - exc_info=True, - ) - raise ressourceException(Reason=str(e)) + self.delete_keypair(key_name=key_name) + return 0 def attach_volume_to_server(self, openstack_id, volume_id): """ From 2a44f44e9a44601c4435cfc3e509558827fd548c Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 16:06:27 +0200 Subject: [PATCH 11/18] feature(ansible): status check --- VirtualMachineService/VirtualMachineHandler.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 04436094..18b9d1c9 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -53,6 +53,8 @@ class VirtualMachineHandler(Iface): """Handler which the PortalClient uses.""" global osi_key_dict + BUILD = "BUILD" + ACTIVE = "ACTIVE" def create_connection(self): """ @@ -597,7 +599,7 @@ def start_server_with_custom_key(self, flavor, image, servername, elixir_id, dis openstack_id = server.to_dict()["id"] global osi_key_dict - osi_key_dict[openstack_id] = dict(key=private_key, name=servername) + osi_key_dict[openstack_id] = dict(key=private_key, name=servername, status=self.BUILD) return {"openstackid": openstack_id, "volumeId": volume_id, 'private_key': private_key} except Exception as e: self.logger.exception("Start Server {1} error:{0}".format(e, servername)) @@ -618,7 +620,7 @@ def create_and_deploy_playbook(self, private_key, play_source, openstack_id): global osi_key_dict playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id]["key"], private_key) playbook.run_it() - + osi_key_dict[openstack_id]["status"] = self.ACTIVE # FOR DEV PURPOSES ONLY! create private key # private_key = NamedTemporaryFile(mode="w+", dir=directory.name, delete=False) # private_key.write("""-----BEGIN RSA PRIVATE KEY----- @@ -775,7 +777,8 @@ def check_server_status(self, openstack_id, diskspace, volume_id): serv = server.to_dict() try: - if serv["status"] == "ACTIVE": + global osi_key_dict + if serv["status"] == self.ACTIVE: host = self.get_server(openstack_id).floating_ip port = self.SSH_PORT @@ -788,7 +791,8 @@ def check_server_status(self, openstack_id, diskspace, volume_id): host = self.add_floating_ip_to_server( openstack_id, self.FLOATING_IP_NETWORK ) - if self.netcat(host, port): + if self.netcat(host, port)\ + and (openstack_id in osi_key_dict and osi_key_dict[openstack_id]["status"] == self.ACTIVE or openstack_id not in osi_key_dict): if diskspace > 0: attached = self.attach_volume_to_server( openstack_id=openstack_id, volume_id=volume_id From 8d7f976a7d4be9f6ab00025028f5c05a40aa60e4 Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 16:07:04 +0200 Subject: [PATCH 12/18] feature(ansible): status check --- VirtualMachineService/VirtualMachineHandler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 18b9d1c9..2daa29b2 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -811,7 +811,7 @@ def check_server_status(self, openstack_id, diskspace, volume_id): return server else: server = self.get_server(openstack_id) - server.status = "BUILD" + server.status = self.BUILD return server except Exception as e: self.logger.exception("Check Status VM {0} error: {1}".format(openstack_id, e)) From 25b569992332681330272f00615a9dee508cbcc6 Mon Sep 17 00:00:00 2001 From: David Weinholz Date: Thu, 4 Jul 2019 16:35:48 +0200 Subject: [PATCH 13/18] added more stati for bioconda --- .../VirtualMachineHandler.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 77c13858..cbfe017a 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -55,6 +55,8 @@ class VirtualMachineHandler(Iface): global osi_key_dict BUILD = "BUILD" ACTIVE = "ACTIVE" + PREPARE_BIOCONDA_BUILD = "PREPARE_BIOCONDA_BUILD" + BUILD_BIOCONDA = "BUILD_BIOCONDA" def create_connection(self): """ @@ -606,7 +608,8 @@ def start_server_with_custom_key(self, flavor, image, servername, elixir_id, dis openstack_id = server.to_dict()["id"] global osi_key_dict - osi_key_dict[openstack_id] = dict(key=private_key, name=servername, status=self.BUILD) + osi_key_dict[openstack_id] = dict(key=private_key, name=servername, + status=self.PREPARE_BIOCONDA_BUILD) return {"openstackid": openstack_id, "volumeId": volume_id, 'private_key': private_key} except Exception as e: self.delete_keypair(key_name=servername) @@ -626,6 +629,7 @@ def create_and_deploy_playbook(self, private_key, play_source, openstack_id): # ip = fields["IP"] # port = fields["PORT"] global osi_key_dict + osi_key_dict[openstack_id]["status"] = self.BUILD key_name = osi_key_dict[openstack_id]['name'] playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id]["key"], private_key) @@ -781,19 +785,29 @@ def check_server_status(self, openstack_id, diskspace, volume_id): host = self.add_floating_ip_to_server( openstack_id, self.FLOATING_IP_NETWORK ) - if self.netcat(host, port)\ - and (openstack_id in osi_key_dict and osi_key_dict[openstack_id]["status"] == self.ACTIVE or openstack_id not in osi_key_dict): + if self.netcat(host, port): + server = self.get_server(openstack_id) + if diskspace > 0: attached = self.attach_volume_to_server( openstack_id=openstack_id, volume_id=volume_id ) if attached is False: - server = self.get_server(openstack_id) self.delete_server(openstack_id=openstack_id) server.status = "DESTROYED" return server - return self.get_server(openstack_id) + if openstack_id in osi_key_dict: + if osi_key_dict[openstack_id][ + "status"] == self.PREPARE_BIOCONDA_BUILD: + server.status = self.PREPARE_BIOCONDA_BUILD + return server + elif osi_key_dict[openstack_id][ + "status"] == self.BUILD_BIOCONDA: + server.status = self.BUILD_BIOCONDA + return server + else: + return server return self.get_server(openstack_id) else: server = self.get_server(openstack_id) From f5b44471784497e547318a65d2f55c10e7886bdf Mon Sep 17 00:00:00 2001 From: David Weinholz Date: Thu, 4 Jul 2019 17:37:51 +0200 Subject: [PATCH 14/18] added biocondata status --- VirtualMachineService/VirtualMachineHandler.py | 4 +++- VirtualMachineService/ancon/BiocondaPlaybook.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index cbfe017a..5d586407 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -629,10 +629,11 @@ def create_and_deploy_playbook(self, private_key, play_source, openstack_id): # ip = fields["IP"] # port = fields["PORT"] global osi_key_dict - osi_key_dict[openstack_id]["status"] = self.BUILD key_name = osi_key_dict[openstack_id]['name'] playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id]["key"], private_key) + osi_key_dict[openstack_id]["status"] = self.BUILD_BIOCONDA + playbook.run_it() osi_key_dict[openstack_id]["status"] = self.ACTIVE # FOR DEV PURPOSES ONLY! create private key @@ -797,6 +798,7 @@ def check_server_status(self, openstack_id, diskspace, volume_id): self.delete_server(openstack_id=openstack_id) server.status = "DESTROYED" return server + if openstack_id in osi_key_dict: if osi_key_dict[openstack_id][ "status"] == self.PREPARE_BIOCONDA_BUILD: diff --git a/VirtualMachineService/ancon/BiocondaPlaybook.py b/VirtualMachineService/ancon/BiocondaPlaybook.py index 8e27526d..dca56dae 100644 --- a/VirtualMachineService/ancon/BiocondaPlaybook.py +++ b/VirtualMachineService/ancon/BiocondaPlaybook.py @@ -34,7 +34,7 @@ def __init__(self, ip, port, play_source, osi_private_key, public_key): yaml_exec.dump(data, variables) def run_it(self): - command_string = "/usr/local/bin/ansible-playbook -i " + self.inventory.name + " " + self.directory.name + "/bioconda.yml" + command_string = "/usr/local/bin/ansible-playbook -vvv -i " + self.inventory.name + " " + self.directory.name + "/bioconda.yml" command_string = shlex.split(command_string) process = subprocess.run(command_string, stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True) self.directory.cleanup() From 7912ae48a2cb5d59b2efe81a27bfd5ae79ba7d47 Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 21:08:05 +0200 Subject: [PATCH 15/18] delete commented out code --- .../VirtualMachineHandler.py | 63 ------------------- 1 file changed, 63 deletions(-) diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 5d586407..33b28ab2 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -617,78 +617,15 @@ def start_server_with_custom_key(self, flavor, image, servername, elixir_id, dis return {} def create_and_deploy_playbook(self, private_key, play_source, openstack_id): - # create temporary directory and copy bioconda.yml and variables.yml there - # ancon_dir = "/code/VirtualMachineService/ancon" - # playbooks_dir = ancon_dir + "/playbooks" - # directory = TemporaryDirectory(dir=ancon_dir) - # shutil.copy(playbooks_dir+"/bioconda.yml", directory.name) - # shutil.copy(playbooks_dir+"/variables.yml", directory.name) - # get ip and port for inventory fields = self.get_ip_ports(openstack_id=openstack_id) - # ip = fields["IP"] - # port = fields["PORT"] global osi_key_dict key_name = osi_key_dict[openstack_id]['name'] playbook = BiocondaPlaybook(fields["IP"], fields["PORT"], play_source, osi_key_dict[openstack_id]["key"], private_key) osi_key_dict[openstack_id]["status"] = self.BUILD_BIOCONDA - playbook.run_it() osi_key_dict[openstack_id]["status"] = self.ACTIVE - # FOR DEV PURPOSES ONLY! create private key - # private_key = NamedTemporaryFile(mode="w+", dir=directory.name, delete=False) - # private_key.write("""-----BEGIN RSA PRIVATE KEY----- - # MIIEpAIBAAKCAQEAvO3PMJojazQj1XljbTph404C74oQR3qRsWzhUyBERVlRsdJi - # Wv0i+LPOmFKDyZ5e7c5YHoPaqItgfc4L49f+v0TcwYyAJth85jv6VF4qz+hEzmi5 - # GlR3IXmUCk8JuyuU+o6oHqxr8RD3lBT76Lwu2FIN5OqXZlMMDwzkWex/8rrFVGyM - # JL1OSpnt6pspPdp7Nlxy7rakSprfalqhRlK0BWf/jkVx+VZwMQVZNNUlLaDIAzyO - # 03aEbnCAZVG+6OXbr9hZwwZ9WY+XfA013uGandntQQdShdop/nm6NirUyQE9gtzM - # omeoGt+u/YK1/2MbOouUlx2aEXkFWIJWrZd6owIDAQABAoIBAQCmP0jzRp9mJVJm - # 9dMk+ZvLjgkNSds7WsK7csjwAdOxhoBZznxX/qn4WRixduKa1u5HqixmZbZSW5sD - # +P0DeDyliG4NLppSFGwLmLmV5escWhG54/MGFU9jOH2peJVii14kAMY1f5nYXgrN - # 1o045eb+2W16g2fIVcmlsL17151bM7WD5p+Jsm6zxIR23pV+NAgY6rt/Kdy+y5O1 - # rh+oQrwEx0+k5ig6DIRkfgBvAKwreqaaPRHMryxOOR/CRxB6zLyoyatCEjCWDLcl - # cii0Zz95c6msHVBeZ4x2HeRulitLdvD9f4oYaojw277wg3mBDMWq2GZMc7xHl+6f - # 14cAsp0BAoGBAN2yXd9VH5ptiEwi8aKqc6RxPwiofCjnkfwAUiBeiHKbSMidnDDp - # +N4Azt1KFCIMWWTkxr/h1LHbhTBdJrLyR+HKHQ9qyJrqm3vNmk0WYW6PiX5LdlHL - # cKaRlyfjt7BGfjBetnrapcPuxQEQ1VIRZgXTaY5MIhrXgsUYH/GFyLSBAoGBANop - # egmuOHm0pk+DJyfivozhhYmWoNlA8x3aqdnZuseMonpLROBEtQrF0YsHHFHiRUvG - # K/6RYzVudpKSmB8grmmGmYNMwppD9ZJ55+/1cnFCQFmU1oENH1GDZhLmNTeinDXV - # LgFhCH+bR3Lp/6kdlEpXVXE9sinue4V/me3m3E0jAoGAKoj0VcshOyHUyrbRoaIO - # efh4XZLl73sumSj+mNNKXqLIfiUvOHtLklyZU//IiRfRdvgl4d7UTiOOFE9rA15U - # yE9c7/5O6tokZsZ12mB25R2JBcA4vUzJGkxIshCQx7Netq0VWdDliQggqCmwpARO - # jMOZNwIIcRn0LxiH2HEQpwECgYEAzzvDD1sNjp7JtJITKdI7y7uWjAInvPfzeRJz - # cdtfj5rJ5H2Haboad6c9y2Dvx+C2jqoqtGEK6oCJ5eWW10rGIruXK6BI4x1XMtLW - # PZzcHzYdxnqZ4HDEpTu6RI2lU7oFxSVB1FGGLyEjl8cr8kuEx7F6Gl3O1gISF9gE - # MnawIh0CgYBSmrR8kHDkHLWbPkKhH93qrJxbo0vSsk+F77Zb+0jA5yLk4SK8jhcx - # /p8iKdiMxnfR6BZyBUYMMW6AXx0Wg+iRQflka+/ugJUFyvELPm2s4ENRY68pTLj8 - # Zc+swBO6T87o29E6e4NLDtQ+h9MsoImfKOLRNKSZttOzfjW3MXqRMg== - # -----END RSA PRIVATE KEY-----""") - # private_key.close() - # - # # create inventory and add the to-be-installed tools to the variables.yml - # inventory = NamedTemporaryFile(mode="w+", dir=directory.name, delete=False) - # inventory_string = "[vm]\n" + ip + ":" + port + " ansible_user=ubuntu " \ - # "ansible_ssh_private_key_file=" + private_key.name - # inventory.write(inventory_string) - # inventory.close() - # yaml_exec = ruamel.yaml.YAML() - # #self.logger.info(play_source) - # #self.logger.info(play_source.strip('\"')) - # play_source = play_source.strip('\"') - # with open(directory.name + "/variables.yml", mode='r+') as variables: - # data = yaml_exec.load(variables) - # data["tools"]["string_line"] = play_source - # #self.logger.info(data["tools"]["string_line"]) - # yaml_exec.dump(data, variables) - # - # # start a subprocess which executes ansible-playbook with bioconda.yml and the inventory - # command_string = "/usr/local/bin/ansible-playbook -i " + inventory.name + " " + directory.name + "/bioconda.yml" - # command_string = shlex.split(command_string) - # process = subprocess.run(command_string, stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True) - # directory.cleanup() - self.delete_keypair(key_name=key_name) return 0 From 77b2516cb3e4cca01733c66e84d3f4787d23aa5a Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 21:11:22 +0200 Subject: [PATCH 16/18] delete unneccessary callback object --- VirtualMachineService/ancon/callback.py | 199 ------------------------ 1 file changed, 199 deletions(-) delete mode 100644 VirtualMachineService/ancon/callback.py diff --git a/VirtualMachineService/ancon/callback.py b/VirtualMachineService/ancon/callback.py deleted file mode 100644 index 87ac283a..00000000 --- a/VirtualMachineService/ancon/callback.py +++ /dev/null @@ -1,199 +0,0 @@ -from datetime import datetime -from ansible.plugins.callback import CallbackBase -import logging -import os -#from django.conf import settings - - -class PlayLogger: - """Store log output in a single object. - We create a new object per Ansible run - """ - def __init__(self): - self.log = '' - self.runtime = 0 - - def append(self, log_line): - """append to log""" - self.log += log_line+"\n\n" - - def banner(self, msg): - """Output Trailing Stars""" - width = 78 - len(msg) - if width < 3: - width = 3 - filler = "*" * width - return "\n%s %s " % (msg, filler) - - -class CallbackModule(CallbackBase): - """ - Reference: https://github.com/ansible/ansible/blob/v2.0.0.2-1/lib/ansible/plugins/callback/default.py - """ - - CALLBACK_VERSION = 2.0 - CALLBACK_TYPE = 'stored' - CALLBACK_NAME = 'database' - - def __init__(self): - super(CallbackModule, self).__init__() - self.logger = PlayLogger() - self.start_time = datetime.now() - - def v2_runner_on_failed(self, result, ignore_errors=False): - delegated_vars = result._result.get('_ansible_delegated_vars', None) - - # Catch an exception - # This may never be called because default handler deletes - # the exception, since Ansible thinks it knows better - if 'exception' in result._result: - # Extract the error message and log it - error = result._result['exception'].strip().split('\n')[-1] - self.logger.append(error) - - # Remove the exception from the result so it's not shown every time - del result._result['exception'] - - # Else log the reason for the failure - if result._task.loop and 'results' in result._result: - self._process_items(result) # item_on_failed, item_on_skipped, item_on_ok - else: - if delegated_vars: - self.logger.append("fatal: [%s -> %s]: FAILED! => %s" % (result._host.get_name(), - delegated_vars['ansible_host'], - self._dump_results(result._result))) - else: - self.logger.append("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result))) - - def v2_runner_on_ok(self, result): - self._clean_results(result._result, result._task.action) - delegated_vars = result._result.get('_ansible_delegated_vars', None) - if result._task.action == 'include': - return - elif result._result.get('changed', False): - if delegated_vars: - msg = "changed: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) - else: - msg = "changed: [%s]" % result._host.get_name() - else: - if delegated_vars: - msg = "ok: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) - else: - msg = "ok: [%s]" % result._host.get_name() - - if result._task.loop and 'results' in result._result: - self._process_items(result) # item_on_failed, item_on_skipped, item_on_ok - else: - self.logger.append(msg) - - def v2_runner_on_skipped(self, result): - if result._task.loop and 'results' in result._result: - self._process_items(result) # item_on_failed, item_on_skipped, item_on_ok - else: - msg = "skipping: [%s]" % result._host.get_name() - self.logger.append(msg) - - def v2_runner_on_unreachable(self, result): - delegated_vars = result._result.get('_ansible_delegated_vars', None) - if delegated_vars: - self.logger.append("fatal: [%s -> %s]: UNREACHABLE! => %s" % (result._host.get_name(), - delegated_vars['ansible_host'], - self._dump_results(result._result))) - else: - self.logger.append("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), - self._dump_results(result._result))) - - def v2_runner_on_no_hosts(self, task): - self.logger.append("skipping: no hosts matched") - - def v2_playbook_on_task_start(self, task, is_conditional): - self.logger.append("TASK [%s]" % task.get_name().strip()) - - def v2_playbook_on_play_start(self, play): - name = play.get_name().strip() - if not name: - msg = "PLAY" - else: - msg = "PLAY [%s]" % name - - self.logger.append(msg) - - def v2_playbook_item_on_ok(self, result): - delegated_vars = result._result.get('_ansible_delegated_vars', None) - if result._task.action == 'include': - return - elif result._result.get('changed', False): - if delegated_vars: - msg = "changed: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) - else: - msg = "changed: [%s]" % result._host.get_name() - else: - if delegated_vars: - msg = "ok: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) - else: - msg = "ok: [%s]" % result._host.get_name() - - msg += " => (item=%s)" % (result._result['item']) - - self.logger.append(msg) - - def v2_playbook_item_on_failed(self, result): - delegated_vars = result._result.get('_ansible_delegated_vars', None) - if 'exception' in result._result: - # Extract the error message and log it - error = result._result['exception'].strip().split('\n')[-1] - self.logger.append(error) - - # Remove the exception from the result so it's not shown every time - del result._result['exception'] - - if delegated_vars: - self.logger.append("failed: [%s -> %s] => (item=%s) => %s" % (result._host.get_name(), - delegated_vars['ansible_host'], - result._result['item'], - self._dump_results(result._result))) - else: - self.logger.append("failed: [%s] => (item=%s) => %s" % (result._host.get_name(), - result._result['item'], - self._dump_results(result._result))) - - def v2_playbook_item_on_skipped(self, result): - msg = "skipping: [%s] => (item=%s) " % (result._host.get_name(), result._result['item']) - self.logger.append(msg) - - def v2_playbook_on_stats(self, stats): - self.logger.runtimee = datetime.now() - self.start_time - #self.logger.runtime = run_time.seconds # returns an int, unlike run_time.total_seconds() - - hosts = sorted(stats.processed.keys()) - for h in hosts: - t = stats.summarize(h) - - msg = "PLAY RECAP [%s] : %s %s %s %s %s" % ( - h, - "ok: %s" % (t['ok']), - "changed: %s" % (t['changed']), - "unreachable: %s" % (t['unreachable']), - "skipped: %s" % (t['skipped']), - "failed: %s" % (t['failures']), - ) - - self.logger.append(msg) - - def record_logs(self): - """ - Special callback added to this callback plugin - Called by Runner objet - :param user_id: - :return: - """ - - logger = logging.getLogger(__name__) - fh = logging.FileHandler('log/ansible_playbook.log', delay=True) - formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(message)s') - fh.setFormatter(formatter) - logger.addHandler(fh) - print(self.start_time) - print(datetime.now() - self.start_time) - return logger.log(level=1000, msg=self.logger.log + str(self.logger.runtime)) From f97f5c3780dec99c947ada7302f1a79e77c88099 Mon Sep 17 00:00:00 2001 From: Ewgenij Katchko Date: Thu, 4 Jul 2019 22:40:57 +0200 Subject: [PATCH 17/18] feature(ansible): output logged to file --- .../ancon/BiocondaPlaybook.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/VirtualMachineService/ancon/BiocondaPlaybook.py b/VirtualMachineService/ancon/BiocondaPlaybook.py index dca56dae..760c869b 100644 --- a/VirtualMachineService/ancon/BiocondaPlaybook.py +++ b/VirtualMachineService/ancon/BiocondaPlaybook.py @@ -1,3 +1,4 @@ +import logging import shlex import shutil import sys @@ -9,6 +10,23 @@ class BiocondaPlaybook(object): def __init__(self, ip, port, play_source, osi_private_key, public_key): + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.DEBUG) + # create file handler which logs even debug messages + self.fh = logging.FileHandler("log/portal_client_ansible.log") + self.fh.setLevel(logging.DEBUG) + self.ch = logging.StreamHandler() + self.ch.setLevel(logging.DEBUG) + # create console handler with a higher log level + # create formatter and add it to the handlers + self.formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + self.fh.setFormatter(self.formatter) + self.ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(self.fh) + self.logger.addHandler(self.ch) self.ancon_dir = "/code/VirtualMachineService/ancon" self.playbooks_dir = self.ancon_dir + "/playbooks" self.directory = TemporaryDirectory(dir=self.ancon_dir) @@ -36,5 +54,7 @@ def __init__(self, ip, port, play_source, osi_private_key, public_key): def run_it(self): command_string = "/usr/local/bin/ansible-playbook -vvv -i " + self.inventory.name + " " + self.directory.name + "/bioconda.yml" command_string = shlex.split(command_string) - process = subprocess.run(command_string, stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True) + process = subprocess.run(command_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + self.logger.log(level=20, msg=process.stdout) + self.logger.log(level=50, msg=process.stderr) self.directory.cleanup() From 070c23ac343d1a19207b301a0132dfed74146852 Mon Sep 17 00:00:00 2001 From: David Weinholz Date: Wed, 10 Jul 2019 22:47:41 +0200 Subject: [PATCH 18/18] updated changelog --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 064e1178..8a5b3fb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## (2019-07-10) + + +#### Features + +* **Snapshot:** addeed description ([e7570b12](e7570b12)) + +#### Bug Fixes + +* **ansible:** fix string.strip error ([dafd8045](dafd8045)) + + ## (2019-06-18) #### Refactor