diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index fb8b82da..2d55a32e 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -168,6 +168,7 @@ def __init__(self, config): try: self.BIBIGRID_URL = cfg["bibigrid"]["bibigrid_url"] self.SUB_NETWORK = cfg["bibigrid"]["sub_network"] + self.BIBIGRID_MODES = cfg["bibigrid"]["bibigrid_modes"] self.logger.info( msg="Bibigrd url loaded: {0}".format(self.BIBIGRID_URL) ) @@ -1697,6 +1698,23 @@ def add_udp_security_group(self, server_id): return True + def detach_ip_from_server(self, server_id, floating_ip): + self.logger.info( + "Detaching floating ip {} from server {}".format(floating_ip, server_id) + ) + try: + self.conn.compute.remove_floating_ip_from_server( + server=server_id, address=floating_ip + ) + return True + except Exception: + self.logger.exception( + "Could not detach floating ip {} from server {}".format( + floating_ip, server_id + ) + ) + return False + def get_servers_by_bibigrid_id(self, bibigrid_id): filters = {"bibigrid_id": bibigrid_id, "name": bibigrid_id} servers = self.conn.list_servers(filters=filters) @@ -1740,20 +1758,33 @@ def get_cluster_status(self, cluster_id): response = req.get( url=request_url, json=body, headers=headers, verify=self.PRODUCTION ) - self.logger.info("Cluster {} status: ".format(cluster_id, response.content)) + self.logger.info("Cluster {} status: {} ".format(cluster_id, response.content)) return response.json() - def get_cluster_info(self, cluster_id): + + def bibigrid_available(self): + if not self.BIBIGRID_URL: + return False + try: + self.get_clusters_info() + return True + except Exception: + self.logger.exception("Bibigrid is offline") + return False + + def get_clusters_info(self): headers = {"content-Type": "application/json"} body = {"mode": "openstack"} request_url = self.BIBIGRID_URL + "list" - self.logger.info(request_url) - response = req.get( url=request_url, json=body, headers=headers, verify=self.PRODUCTION ) self.logger.info(response.json()) infos = response.json()["info"] + return infos + + def get_cluster_info(self, cluster_id): + infos = self.get_clusters_info() for info in infos: self.logger.info(cluster_id) self.logger.info(info) @@ -1802,7 +1833,10 @@ def start_cluster(self, public_key, master_instance, worker_instances, user): "availabilityZone": self.AVAIALABILITY_ZONE, "masterInstance": master_instance, "workerInstances": wI, + "useMasterWithPublicIp": False } + for mode in self.BIBIGRID_MODES: + body.update({mode: True}) request_url = self.BIBIGRID_URL + "create" response = req.post( url=request_url, json=body, headers=headers, verify=self.PRODUCTION diff --git a/VirtualMachineService/VirtualMachineService-remote b/VirtualMachineService/VirtualMachineService-remote index dc433081..668a78e7 100755 --- a/VirtualMachineService/VirtualMachineService-remote +++ b/VirtualMachineService/VirtualMachineService-remote @@ -43,6 +43,8 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' string add_floating_ip_to_server(string openstack_id, string network)') print(' bool create_connection(string username, string password, string auth_url, string user_domain_name, string project_domain_name)') print(' start_server_without_playbook(string flavor, string image, string public_key, string servername, metadata, bool https, bool http, resenv, volume_ids_path_new, volume_ids_path_attach)') + print(' bool bibigrid_available()') + print(' bool detach_ip_from_server(string server_id, string floating_ip)') print(' start_server_with_mounted_volume(string flavor, string image, string public_key, string servername, metadata, bool https, bool http, resenv, volume_ids_path_new, volume_ids_path_attach)') print(' start_server(string flavor, string image, string public_key, string servername, metadata, string diskspace, string volumename, bool https, bool http, resenv)') print(' start_server_with_custom_key(string flavor, string image, string servername, metadata, bool http, bool https, resenv, volume_ids_path_new, volume_ids_path_attach)') @@ -275,6 +277,18 @@ elif cmd == 'start_server_without_playbook': sys.exit(1) pp.pprint(client.start_server_without_playbook(args[0], args[1], args[2], args[3], eval(args[4]), eval(args[5]), eval(args[6]), eval(args[7]), eval(args[8]), eval(args[9]),)) +elif cmd == 'bibigrid_available': + if len(args) != 0: + print('bibigrid_available requires 0 args') + sys.exit(1) + pp.pprint(client.bibigrid_available()) + +elif cmd == 'detach_ip_from_server': + if len(args) != 2: + print('detach_ip_from_server requires 2 args') + sys.exit(1) + pp.pprint(client.detach_ip_from_server(args[0], args[1],)) + elif cmd == 'start_server_with_mounted_volume': if len(args) != 10: print('start_server_with_mounted_volume requires 10 args') diff --git a/VirtualMachineService/VirtualMachineService.py b/VirtualMachineService/VirtualMachineService.py index 002ccd67..8315b5d2 100644 --- a/VirtualMachineService/VirtualMachineService.py +++ b/VirtualMachineService/VirtualMachineService.py @@ -233,6 +233,18 @@ def start_server_without_playbook( """ pass + def bibigrid_available(self): + pass + + def detach_ip_from_server(self, server_id, floating_ip): + """ + Parameters: + - server_id + - floating_ip + + """ + pass + def start_server_with_mounted_volume( self, flavor, @@ -1484,6 +1496,76 @@ def recv_start_server_without_playbook(self): "start_server_without_playbook failed: unknown result", ) + def bibigrid_available(self): + self.send_bibigrid_available() + return self.recv_bibigrid_available() + + def send_bibigrid_available(self): + self._oprot.writeMessageBegin( + "bibigrid_available", TMessageType.CALL, self._seqid + ) + args = bibigrid_available_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_bibigrid_available(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = bibigrid_available_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException( + TApplicationException.MISSING_RESULT, + "bibigrid_available failed: unknown result", + ) + + def detach_ip_from_server(self, server_id, floating_ip): + """ + Parameters: + - server_id + - floating_ip + + """ + self.send_detach_ip_from_server(server_id, floating_ip) + return self.recv_detach_ip_from_server() + + def send_detach_ip_from_server(self, server_id, floating_ip): + self._oprot.writeMessageBegin( + "detach_ip_from_server", TMessageType.CALL, self._seqid + ) + args = detach_ip_from_server_args() + args.server_id = server_id + args.floating_ip = floating_ip + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_detach_ip_from_server(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = detach_ip_from_server_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException( + TApplicationException.MISSING_RESULT, + "detach_ip_from_server failed: unknown result", + ) + def start_server_with_mounted_volume( self, flavor, @@ -3218,6 +3300,10 @@ def __init__(self, handler): self._processMap[ "start_server_without_playbook" ] = Processor.process_start_server_without_playbook + self._processMap["bibigrid_available"] = Processor.process_bibigrid_available + self._processMap[ + "detach_ip_from_server" + ] = Processor.process_detach_ip_from_server self._processMap[ "start_server_with_mounted_volume" ] = Processor.process_start_server_with_mounted_volume @@ -3834,6 +3920,58 @@ def process_start_server_without_playbook(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_bibigrid_available(self, seqid, iprot, oprot): + args = bibigrid_available_args() + args.read(iprot) + iprot.readMessageEnd() + result = bibigrid_available_result() + try: + result.success = self._handler.bibigrid_available() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException( + TApplicationException.INTERNAL_ERROR, "Internal error" + ) + oprot.writeMessageBegin("bibigrid_available", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_detach_ip_from_server(self, seqid, iprot, oprot): + args = detach_ip_from_server_args() + args.read(iprot) + iprot.readMessageEnd() + result = detach_ip_from_server_result() + try: + result.success = self._handler.detach_ip_from_server( + args.server_id, args.floating_ip + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException( + TApplicationException.INTERNAL_ERROR, "Internal error" + ) + oprot.writeMessageBegin("detach_ip_from_server", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_start_server_with_mounted_volume(self, seqid, iprot, oprot): args = start_server_with_mounted_volume_args() args.read(iprot) @@ -8474,6 +8612,287 @@ def __ne__(self, other): ) +class bibigrid_available_args(object): + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("bibigrid_available_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(bibigrid_available_args) +bibigrid_available_args.thrift_spec = () + + +class bibigrid_available_result(object): + """ + Attributes: + - success + + """ + + def __init__( + self, success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("bibigrid_available_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(bibigrid_available_result) +bibigrid_available_result.thrift_spec = ((0, TType.BOOL, "success", None, None,),) # 0 + + +class detach_ip_from_server_args(object): + """ + Attributes: + - server_id + - floating_ip + + """ + + def __init__( + self, server_id=None, floating_ip=None, + ): + self.server_id = server_id + self.floating_ip = floating_ip + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.server_id = ( + iprot.readString().decode("utf-8") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.floating_ip = ( + iprot.readString().decode("utf-8") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("detach_ip_from_server_args") + if self.server_id is not None: + oprot.writeFieldBegin("server_id", TType.STRING, 1) + oprot.writeString( + self.server_id.encode("utf-8") + if sys.version_info[0] == 2 + else self.server_id + ) + oprot.writeFieldEnd() + if self.floating_ip is not None: + oprot.writeFieldBegin("floating_ip", TType.STRING, 2) + oprot.writeString( + self.floating_ip.encode("utf-8") + if sys.version_info[0] == 2 + else self.floating_ip + ) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(detach_ip_from_server_args) +detach_ip_from_server_args.thrift_spec = ( + None, # 0 + (1, TType.STRING, "server_id", "UTF8", None,), # 1 + (2, TType.STRING, "floating_ip", "UTF8", None,), # 2 +) + + +class detach_ip_from_server_result(object): + """ + Attributes: + - success + + """ + + def __init__( + self, success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("detach_ip_from_server_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(detach_ip_from_server_result) +detach_ip_from_server_result.thrift_spec = ( + (0, TType.BOOL, "success", None, None,), # 0 +) + + class start_server_with_mounted_volume_args(object): """ Attributes: diff --git a/VirtualMachineService/config/config.yml b/VirtualMachineService/config/config.yml index 69c401c9..58a5172b 100644 --- a/VirtualMachineService/config/config.yml +++ b/VirtualMachineService/config/config.yml @@ -32,6 +32,9 @@ bibigrid: # Url for Bibigrid API bibigrid_url: https://172.21.0.1:8443/bibigrid/ sub_network: portalexternalsubnetwork + bibigrid_modes: + - slurm + forc: forc_url: https://proxy-dev.bi.denbi.de:5000/ forc_allowed: diff --git a/portal_client.thrift b/portal_client.thrift index 32d65ab9..da7dc64e 100644 --- a/portal_client.thrift +++ b/portal_client.thrift @@ -368,7 +368,8 @@ service VirtualMachineService { throws (1:nameException e,2:ressourceException r,3:serverNotFoundException s,4: networkNotFoundException n,5:imageNotFoundException i,6:flavorNotFoundException f,7:otherException o) - + bool bibigrid_available() + bool detach_ip_from_server(1:string server_id,2:string floating_ip) map start_server_with_mounted_volume( /** Name of the Flavor to use.*/ diff --git a/gateway/gateway_TCP.sh b/scripts/gateway/gateway_TCP.sh similarity index 100% rename from gateway/gateway_TCP.sh rename to scripts/gateway/gateway_TCP.sh diff --git a/gateway/gateway_UDP.sh b/scripts/gateway/gateway_UDP.sh similarity index 100% rename from gateway/gateway_UDP.sh rename to scripts/gateway/gateway_UDP.sh diff --git a/scripts/generate_new_test_pems.sh b/scripts/generate_new_test_pems.sh new file mode 100644 index 00000000..fddf37fe --- /dev/null +++ b/scripts/generate_new_test_pems.sh @@ -0,0 +1,19 @@ +#!/usr/bin/expect +echo Creating new dir "new_pem" +mkdir new_pem +cd new_pem +openssl req -new -x509 -nodes -days 3000 -out server.crt -keyout server.key -subj "/C=DE/ST=./L=./O=./CN=$1" + +openssl x509 -in server.crt -text > CA.pem +cat server.crt server.key > server.pem +openssl pkcs12 -export -clcerts -in server.crt -inkey server.key -out server.p12 -passout pass:thrift +openssl genrsa -out client.key +openssl req -new -nodes -key client.key -out client.csr -subj "/C=DE/ST=./L=./O=./CN=$1" -passin pass:thrift +openssl x509 -req -days 3000 -in client.csr -CA CA.pem -CAkey server.key -set_serial 01 -out client.crt +openssl pkcs12 -export -clcerts -in client.crt -inkey client.key -out client.p12 -passout pass:thrift +openssl pkcs12 -in client.p12 -out client.pem -clcerts -passin pass:thrift -passout pass:thrift +openssl rsa -in client.pem -out client_no_pass.pem -passin pass:thrift +sed -i '/-----BEGIN ENCRYPTED PRIVATE KEY-----/Q' client.pem +cat client_no_pass.pem >> client.pem +find . -type f -not -name '*.pem' -print0 | xargs -0 rm +rm client_no_pass.pem