diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c68d9ad..de5821b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: check-yaml - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 23.11.0 hooks: - id: black - repo: https://github.com/sondrelg/pep585-upgrade diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml index 5c05257..58aa079 100644 --- a/docker-compose.bibigrid.yml +++ b/docker-compose.bibigrid.yml @@ -18,7 +18,7 @@ services: simplevm_client_redis: container_name: simplevm_client_redis - image: redis:7.2.2 + image: redis:7.2.3 expose: - "6379" networks: @@ -41,7 +41,7 @@ services: # filebeat simplevm_filebeat: - image: docker.elastic.co/beats/filebeat:8.10.4 + image: docker.elastic.co/beats/filebeat:8.11.1 env_file: - .env volumes: diff --git a/docker-compose.dev.bibigrid.yml b/docker-compose.dev.bibigrid.yml index 07ccfcf..66880ca 100644 --- a/docker-compose.dev.bibigrid.yml +++ b/docker-compose.dev.bibigrid.yml @@ -18,7 +18,7 @@ services: client_redis: container_name: simplevm_client_redis - image: redis:7.2.2 + image: redis:7.2.3 expose: - "6379" networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d7159fa..4b06e12 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -18,7 +18,7 @@ services: client_redis: container_name: simplevm_client_redis - image: redis:7.2.2 + image: redis:7.2.3 expose: - "6379" networks: diff --git a/docker-compose.yml b/docker-compose.yml index 50554f0..94e2554 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: simplevm_client_redis: container_name: simplevm_client_redis - image: redis:7.2.2 + image: redis:7.2.3 expose: - "6379" networks: @@ -27,7 +27,7 @@ services: # filebeat simplevm_client_filebeat: container_name: simplevm_client_filebeat - image: docker.elastic.co/beats/filebeat:8.10.4 + image: docker.elastic.co/beats/filebeat:8.11.1 env_file: - .env volumes: diff --git a/portal_client.thrift b/portal_client.thrift index f4e43c7..5c5bb8c 100644 --- a/portal_client.thrift +++ b/portal_client.thrift @@ -233,6 +233,11 @@ exception ServerNotFoundException { 2: string name_or_id } +exception SecurityGroupRuleNotFoundException { + /** Server not found. */ + 1: string message + 2: string name_or_id +} exception FlavorNotFoundException { 1: string message @@ -388,6 +393,12 @@ service VirtualMachineService { void resize_volume(1:string volume_id,2:int size) throws(1:VolumeNotFoundException v) + /** + * Creates/Updates a security group for a vm with a specific port range for a project + */ + string open_port_range_for_vm_in_project(1:int range_start,2:int range_stop,3:string openstack_id,4: string ethertype = "IPv4",5:string protocol ="TCP") throws (1:ServerNotFoundException e,2: DefaultException v,3:OpenStackConflictException o) + + void delete_security_group_rule(1:string openstack_id) throws (1:SecurityGroupRuleNotFoundException e,2:DefaultException f) /** diff --git a/requirements.txt b/requirements.txt index ab4d27a..ad3b6a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ -setuptools==68.2.2 +setuptools==69.0.2 thrift==0.16.0 python-keystoneclient==5.2.0 openstacksdk==2.0.0 deprecated==1.2.14 Click==8.1.7 -ansible==8.5.0 +ansible==5.1.0 flake8==6.1.0 paramiko==2.12.0 -ruamel.yaml==0.18.3 +ruamel.yaml==0.18.5 pyvim==3.0.3 redis==5.0.1 requests==2.31.0 @@ -15,4 +15,4 @@ pyyaml==6.0.1 pre-commit==3.5.0 types-PyYAML==6.0.12.12 sympy==1.12 -types-redis==4.6.0.8 +types-redis==4.6.0.11 diff --git a/simple_vm_client/VirtualMachineHandler.py b/simple_vm_client/VirtualMachineHandler.py index 789d715..547fd58 100644 --- a/simple_vm_client/VirtualMachineHandler.py +++ b/simple_vm_client/VirtualMachineHandler.py @@ -299,6 +299,27 @@ def delete_user_from_backend(self, backend_id: str, user_id: str) -> dict[str, s def get_allowed_templates(self) -> list[ResearchEnvironmentTemplate]: return self.forc_connector.template.get_allowed_templates() + def delete_security_group_rule(self, openstack_id): + return self.openstack_connector.delete_security_group_rule( + openstack_id=openstack_id + ) + + def open_port_range_for_vm_in_project( + self, + range_start, + range_stop, + openstack_id, + ethertype: str = "IPv4", + protocol: str = "TCP", + ) -> str: + return self.openstack_connector.open_port_range_for_vm_in_project( + range_start=range_start, + range_stop=range_stop, + openstack_id=openstack_id, + ethertype=ethertype, + protocol=protocol, + ) + def add_udp_security_group(self, server_id: str) -> None: return self.openstack_connector.add_udp_security_group(server_id=server_id) diff --git a/simple_vm_client/VirtualMachineService-remote b/simple_vm_client/VirtualMachineService-remote index 8419b7f..9c49441 100755 --- a/simple_vm_client/VirtualMachineService-remote +++ b/simple_vm_client/VirtualMachineService-remote @@ -44,6 +44,10 @@ if len(sys.argv) <= 1 or sys.argv[1] == "--help": print(" Volume get_volume(string volume_id)") print(" get_volumes_by_ids( volume_ids)") print(" void resize_volume(string volume_id, int size)") + print( + " string open_port_range_for_vm_in_project(int range_start, int range_stop, string openstack_id, string ethertype, string protocol)" + ) + print(" void delete_security_group_rule(string openstack_id)") print(" void delete_server(string openstack_id)") print( " string start_server(string flavor_name, string image_name, string public_key, string servername, metadata, volume_ids_path_new, volume_ids_path_attach, additional_keys, string research_environment, additional_security_group_ids)" @@ -334,6 +338,30 @@ elif cmd == "resize_volume": ) ) +elif cmd == "open_port_range_for_vm_in_project": + if len(args) != 5: + print("open_port_range_for_vm_in_project requires 5 args") + sys.exit(1) + pp.pprint( + client.open_port_range_for_vm_in_project( + eval(args[0]), + eval(args[1]), + args[2], + args[3], + args[4], + ) + ) + +elif cmd == "delete_security_group_rule": + if len(args) != 1: + print("delete_security_group_rule requires 1 args") + sys.exit(1) + pp.pprint( + client.delete_security_group_rule( + args[0], + ) + ) + elif cmd == "delete_server": if len(args) != 1: print("delete_server requires 1 args") diff --git a/simple_vm_client/VirtualMachineService.py b/simple_vm_client/VirtualMachineService.py index 814e7bf..461f7e0 100644 --- a/simple_vm_client/VirtualMachineService.py +++ b/simple_vm_client/VirtualMachineService.py @@ -148,6 +148,28 @@ def resize_volume(self, volume_id, size): """ + def open_port_range_for_vm_in_project( + self, range_start, range_stop, openstack_id, ethertype, protocol + ): + """ + Creates/Updates a security group for a vm with a specific port range for a project + + Parameters: + - range_start + - range_stop + - openstack_id + - ethertype + - protocol + + """ + + def delete_security_group_rule(self, openstack_id): + """ + Parameters: + - openstack_id + + """ + def delete_server(self, openstack_id): """ Delete server. @@ -1230,6 +1252,99 @@ def recv_resize_volume(self): raise result.v return + def open_port_range_for_vm_in_project( + self, range_start, range_stop, openstack_id, ethertype, protocol + ): + """ + Creates/Updates a security group for a vm with a specific port range for a project + + Parameters: + - range_start + - range_stop + - openstack_id + - ethertype + - protocol + + """ + self.send_open_port_range_for_vm_in_project( + range_start, range_stop, openstack_id, ethertype, protocol + ) + return self.recv_open_port_range_for_vm_in_project() + + def send_open_port_range_for_vm_in_project( + self, range_start, range_stop, openstack_id, ethertype, protocol + ): + self._oprot.writeMessageBegin( + "open_port_range_for_vm_in_project", TMessageType.CALL, self._seqid + ) + args = open_port_range_for_vm_in_project_args() + args.range_start = range_start + args.range_stop = range_stop + args.openstack_id = openstack_id + args.ethertype = ethertype + args.protocol = protocol + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_open_port_range_for_vm_in_project(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = open_port_range_for_vm_in_project_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.e is not None: + raise result.e + if result.v is not None: + raise result.v + if result.o is not None: + raise result.o + raise TApplicationException( + TApplicationException.MISSING_RESULT, + "open_port_range_for_vm_in_project failed: unknown result", + ) + + def delete_security_group_rule(self, openstack_id): + """ + Parameters: + - openstack_id + + """ + self.send_delete_security_group_rule(openstack_id) + self.recv_delete_security_group_rule() + + def send_delete_security_group_rule(self, openstack_id): + self._oprot.writeMessageBegin( + "delete_security_group_rule", TMessageType.CALL, self._seqid + ) + args = delete_security_group_rule_args() + args.openstack_id = openstack_id + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_delete_security_group_rule(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = delete_security_group_rule_result() + result.read(iprot) + iprot.readMessageEnd() + if result.e is not None: + raise result.e + return + def delete_server(self, openstack_id): """ Delete server. @@ -3277,6 +3392,12 @@ def __init__(self, handler): self._processMap["get_volume"] = Processor.process_get_volume self._processMap["get_volumes_by_ids"] = Processor.process_get_volumes_by_ids self._processMap["resize_volume"] = Processor.process_resize_volume + self._processMap[ + "open_port_range_for_vm_in_project" + ] = Processor.process_open_port_range_for_vm_in_project + self._processMap[ + "delete_security_group_rule" + ] = Processor.process_delete_security_group_rule self._processMap["delete_server"] = Processor.process_delete_server self._processMap["start_server"] = Processor.process_start_server self._processMap[ @@ -3805,6 +3926,74 @@ def process_resize_volume(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_open_port_range_for_vm_in_project(self, seqid, iprot, oprot): + args = open_port_range_for_vm_in_project_args() + args.read(iprot) + iprot.readMessageEnd() + result = open_port_range_for_vm_in_project_result() + try: + result.success = self._handler.open_port_range_for_vm_in_project( + args.range_start, + args.range_stop, + args.openstack_id, + args.ethertype, + args.protocol, + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except ServerNotFoundException as e: + msg_type = TMessageType.REPLY + result.e = e + except DefaultException as v: + msg_type = TMessageType.REPLY + result.v = v + except OpenStackConflictException as o: + msg_type = TMessageType.REPLY + result.o = o + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException( + TApplicationException.INTERNAL_ERROR, "Internal error" + ) + oprot.writeMessageBegin("open_port_range_for_vm_in_project", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_delete_security_group_rule(self, seqid, iprot, oprot): + args = delete_security_group_rule_args() + args.read(iprot) + iprot.readMessageEnd() + result = delete_security_group_rule_result() + try: + self._handler.delete_security_group_rule(args.openstack_id) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except SecurityGroupRuleNotFoundException as e: + msg_type = TMessageType.REPLY + result.e = e + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException( + TApplicationException.INTERNAL_ERROR, "Internal error" + ) + oprot.writeMessageBegin("delete_security_group_rule", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_delete_server(self, seqid, iprot, oprot): args = delete_server_args() args.read(iprot) @@ -7887,6 +8076,486 @@ def __ne__(self, other): ) +class open_port_range_for_vm_in_project_args(object): + """ + Attributes: + - range_start + - range_stop + - openstack_id + - ethertype + - protocol + + """ + + def __init__( + self, + range_start=None, + range_stop=None, + openstack_id=None, + ethertype="IPv4", + protocol="TCP", + ): + self.range_start = range_start + self.range_stop = range_stop + self.openstack_id = openstack_id + self.ethertype = ethertype + self.protocol = protocol + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.range_start = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.range_stop = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.openstack_id = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.ethertype = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.protocol = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("open_port_range_for_vm_in_project_args") + if self.range_start is not None: + oprot.writeFieldBegin("range_start", TType.I32, 1) + oprot.writeI32(self.range_start) + oprot.writeFieldEnd() + if self.range_stop is not None: + oprot.writeFieldBegin("range_stop", TType.I32, 2) + oprot.writeI32(self.range_stop) + oprot.writeFieldEnd() + if self.openstack_id is not None: + oprot.writeFieldBegin("openstack_id", TType.STRING, 3) + oprot.writeString( + self.openstack_id.encode("utf-8") + if sys.version_info[0] == 2 + else self.openstack_id + ) + oprot.writeFieldEnd() + if self.ethertype is not None: + oprot.writeFieldBegin("ethertype", TType.STRING, 4) + oprot.writeString( + self.ethertype.encode("utf-8") + if sys.version_info[0] == 2 + else self.ethertype + ) + oprot.writeFieldEnd() + if self.protocol is not None: + oprot.writeFieldBegin("protocol", TType.STRING, 5) + oprot.writeString( + self.protocol.encode("utf-8") + if sys.version_info[0] == 2 + else self.protocol + ) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(open_port_range_for_vm_in_project_args) +open_port_range_for_vm_in_project_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "range_start", + None, + None, + ), # 1 + ( + 2, + TType.I32, + "range_stop", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "openstack_id", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "ethertype", + "UTF8", + "IPv4", + ), # 4 + ( + 5, + TType.STRING, + "protocol", + "UTF8", + "TCP", + ), # 5 +) + + +class open_port_range_for_vm_in_project_result(object): + """ + Attributes: + - success + - e + - v + - o + + """ + + def __init__( + self, + success=None, + e=None, + v=None, + o=None, + ): + self.success = success + self.e = e + self.v = v + self.o = o + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.e = ServerNotFoundException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.v = DefaultException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o = OpenStackConflictException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("open_port_range_for_vm_in_project_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString( + self.success.encode("utf-8") + if sys.version_info[0] == 2 + else self.success + ) + oprot.writeFieldEnd() + if self.e is not None: + oprot.writeFieldBegin("e", TType.STRUCT, 1) + self.e.write(oprot) + oprot.writeFieldEnd() + if self.v is not None: + oprot.writeFieldBegin("v", TType.STRUCT, 2) + self.v.write(oprot) + oprot.writeFieldEnd() + if self.o is not None: + oprot.writeFieldBegin("o", TType.STRUCT, 3) + self.o.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(open_port_range_for_vm_in_project_result) +open_port_range_for_vm_in_project_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 + ( + 1, + TType.STRUCT, + "e", + [ServerNotFoundException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "v", + [DefaultException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o", + [OpenStackConflictException, None], + None, + ), # 3 +) + + +class delete_security_group_rule_args(object): + """ + Attributes: + - openstack_id + + """ + + def __init__( + self, + openstack_id=None, + ): + self.openstack_id = openstack_id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.openstack_id = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("delete_security_group_rule_args") + if self.openstack_id is not None: + oprot.writeFieldBegin("openstack_id", TType.STRING, 1) + oprot.writeString( + self.openstack_id.encode("utf-8") + if sys.version_info[0] == 2 + else self.openstack_id + ) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(delete_security_group_rule_args) +delete_security_group_rule_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "openstack_id", + "UTF8", + None, + ), # 1 +) + + +class delete_security_group_rule_result(object): + """ + Attributes: + - e + + """ + + def __init__( + self, + e=None, + ): + self.e = e + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.e = SecurityGroupRuleNotFoundException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("delete_security_group_rule_result") + if self.e is not None: + oprot.writeFieldBegin("e", TType.STRUCT, 1) + self.e.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(delete_security_group_rule_result) +delete_security_group_rule_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "e", + [SecurityGroupRuleNotFoundException, None], + None, + ), # 1 +) + + class delete_server_args(object): """ Attributes: diff --git a/simple_vm_client/forc_connector/template/plays/change_key.yml b/simple_vm_client/forc_connector/template/plays/change_key.yml index 8d762c0..5a62e5d 100644 --- a/simple_vm_client/forc_connector/template/plays/change_key.yml +++ b/simple_vm_client/forc_connector/template/plays/change_key.yml @@ -1,7 +1,7 @@ - name: Echo Public Key to set - ansible.builtin.command: echo "${{ change_key_vars.key }}" + ansible.builtin.command: echo "{{ change_key_vars.key }}" tags: - - always + - always - name: Set user public Key and remove created public Key authorized_key: user: ubuntu @@ -9,4 +9,4 @@ state: present exclusive: True tags: - - always + - always diff --git a/simple_vm_client/openstack_connector/openstack_connector.py b/simple_vm_client/openstack_connector/openstack_connector.py index 3cf2cdc..cff9d49 100644 --- a/simple_vm_client/openstack_connector/openstack_connector.py +++ b/simple_vm_client/openstack_connector/openstack_connector.py @@ -126,6 +126,9 @@ def load_config_yml(self, config_file: str) -> None: "gateway_security_group_id" ] + def _get_default_security_groups(self): + return self.DEFAULT_SECURITY_GROUPS.copy() + def load_env_config(self) -> None: logger.info("Load environment config: OpenStack") @@ -135,7 +138,7 @@ def load_env_config(self) -> None: sys.exit(1) self.USE_APPLICATION_CREDENTIALS = ( - os.environ.get("USE_APPLICATION_CREDENTIALS", "False").lower() == "true" + os.environ.get("USE_APPLICATION_CREDENTIALS", "False").lower() == "true" ) if self.USE_APPLICATION_CREDENTIALS: @@ -177,15 +180,15 @@ def load_env_config(self) -> None: self.PROJECT_DOMAIN_ID = os.environ["OS_PROJECT_DOMAIN_ID"] def create_server( - self, - name: str, - image_id: str, - flavor_id: str, - network_id: str, - userdata: str, - key_name: str, - metadata: dict[str, str], - security_groups: list[str], + self, + name: str, + image_id: str, + flavor_id: str, + network_id: str, + userdata: str, + key_name: str, + metadata: dict[str, str], + security_groups: list[str], ) -> Server: logger.info( f"Create Server:\n\tname: {name}\n\timage_id:{image_id}\n\tflavor_id:{flavor_id}\n\tmetadata:{metadata}" @@ -226,7 +229,7 @@ def delete_volume(self, volume_id: str) -> None: raise DefaultException(message=e.message) def create_volume_snapshot( - self, volume_id: str, name: str, description: str + self, volume_id: str, name: str, description: str ) -> str: try: logger.info(f"Create Snapshot for Volume {volume_id}") @@ -266,7 +269,7 @@ def delete_volume_snapshot(self, snapshot_id: str) -> None: raise DefaultException(message=e.message) def create_volume_by_source_volume( - self, volume_name: str, metadata: dict[str, str], source_volume_id: str + self, volume_name: str, metadata: dict[str, str], source_volume_id: str ) -> Volume: logger.info(f"Creating volume from source volume with id {source_volume_id}") try: @@ -282,7 +285,7 @@ def create_volume_by_source_volume( raise ResourceNotAvailableException(message=e.message) def create_volume_by_volume_snap( - self, volume_name: str, metadata: dict[str, str], volume_snap_id: str + self, volume_name: str, metadata: dict[str, str], volume_snap_id: str ) -> Volume: logger.info(f"Creating volume from volume snapshot with id {volume_snap_id}") try: @@ -316,7 +319,7 @@ def get_servers_by_ids(self, ids: list[str]) -> list[Server]: return servers def attach_volume_to_server( - self, openstack_id: str, volume_id: str + self, openstack_id: str, volume_id: str ) -> dict[str, str]: server = self.get_server(openstack_id=openstack_id) volume = self.get_volume(name_or_id=volume_id) @@ -357,7 +360,7 @@ def resize_volume(self, volume_id: str, size: int) -> None: raise DefaultException(message=str(e)) def create_volume( - self, volume_name: str, volume_storage: int, metadata: dict[str, str] + self, volume_name: str, volume_storage: int, metadata: dict[str, str] ) -> Volume: logger.info(f"Creating volume with {volume_storage} GB storage") try: @@ -474,9 +477,9 @@ def get_active_image_by_os_version(self, os_version: str, os_distro: str) -> Ima image_os_distro = metadata.get("os_distro", None) base_image_ref = metadata.get("base_image_ref", None) if ( - os_version == image_os_version - and image.status == "active" - and base_image_ref is None + os_version == image_os_version + and image.status == "active" + and base_image_ref is None ): if os_distro and os_distro == image_os_distro: return image @@ -488,11 +491,11 @@ def get_active_image_by_os_version(self, os_version: str, os_distro: str) -> Ima ) def get_image( - self, - name_or_id: str, - replace_inactive: bool = False, - ignore_not_active: bool = False, - ignore_not_found: bool = False, + self, + name_or_id: str, + replace_inactive: bool = False, + ignore_not_active: bool = False, + ignore_not_found: bool = False, ) -> Image: logger.info(f"Get Image {name_or_id}") @@ -516,12 +519,12 @@ def get_image( return image def create_snapshot( - self, - openstack_id: str, - name: str, - username: str, - base_tags: list[str], - description: str, + self, + openstack_id: str, + name: str, + username: str, + base_tags: list[str], + description: str, ) -> str: logger.info( f"Create Snapshot from Instance {openstack_id} with name {name} for {username}" @@ -633,9 +636,9 @@ def get_gateway_ip(self) -> dict[str, str]: return {"gateway_ip": self.GATEWAY_IP} def create_mount_init_script( - self, - new_volumes: list[dict[str, str]] = None, # type: ignore - attach_volumes: list[dict[str, str]] = None, # type: ignore + self, + new_volumes: list[dict[str, str]] = None, # type: ignore + attach_volumes: list[dict[str, str]] = None, # type: ignore ) -> str: logger.info(f"Create init script for volume ids:{new_volumes}") if not new_volumes and not attach_volumes: @@ -704,14 +707,74 @@ def create_or_get_default_ssh_security_group(self): description="Default SSH SimpleVM Security Group", ) + def delete_security_group_rule(self, openstack_id): + logger.info(f"Delete Security Group Rule -- {openstack_id}") + deleted = self.openstack_connection.delete_security_group_rule( + rule_id=openstack_id + ) + logger.info(f"Delete Result -- {deleted}") + if not deleted: + raise DefaultException( + message=f"Could not delete security group rule - {openstack_id}" + ) + + def open_port_range_for_vm_in_project( + self, range_start, range_stop, openstack_id, ethertype="IPV4", protocol="TCP" + ): + server: Server = self.openstack_connection.get_server_by_id(id=openstack_id) + if server is None: + logger.exception(f"Instance {openstack_id} not found") + raise ServerNotFoundException( + message=f"Instance {openstack_id} not found", + name_or_id=openstack_id, + ) + project_name = server.metadata.get("project_name") + project_id = server.metadata.get("project_id") + + project_security_group = self.get_or_create_project_security_group( + project_name=project_name, project_id=project_id + ) + vm_security_group = self.get_or_create_vm_security_group( + openstack_id=openstack_id + ) + current_vm_security_group_names = [ + sec["name"] for sec in server["security_groups"] + ] + if openstack_id not in current_vm_security_group_names: + self.openstack_connection.add_server_security_groups( + server=server, security_groups=[vm_security_group] + ) + if ethertype not in ["IPv4", "IPv6"]: + raise DefaultException( + message=f"Type {ethertype} does not exist for security group rules" + ) + + try: + security_rule = self.openstack_connection.create_security_group_rule( + direction="ingress", + ethertype=ethertype, + protocol=protocol, + port_range_max=range_stop, + port_range_min=range_start, + secgroup_name_or_id=vm_security_group, + remote_group_id=project_security_group, + ) + return security_rule["id"] + + except ConflictException as e: + logger.exception( + f"Could not create security group rule for instance {openstack_id}" + ) + raise OpenStackConflictException(message=e.message) + def create_security_group( - self, - name: str, - udp_port: int = None, # type: ignore - ssh: bool = True, - udp: bool = False, - description: str = "", - research_environment_metadata: ResearchEnvironmentMetadata = None, + self, + name: str, + udp_port: int = None, # type: ignore + ssh: bool = True, + udp: bool = False, + description: str = "", + research_environment_metadata: ResearchEnvironmentMetadata = None, ) -> SecurityGroup: logger.info(f"Create new security group {name}") sec: SecurityGroup = self.openstack_connection.get_security_group( @@ -818,7 +881,7 @@ def is_security_group_in_use(self, security_group_id): return False def get_or_create_research_environment_security_group( - self, resenv_metadata: ResearchEnvironmentMetadata + self, resenv_metadata: ResearchEnvironmentMetadata ): if not resenv_metadata.needs_forc_support: return None @@ -851,6 +914,18 @@ def get_or_create_research_environment_security_group( ) return new_security_group["id"] + def get_or_create_vm_security_group(self, openstack_id): + logger.info(f"Check if Security Group for vm - [{openstack_id}] exists... ") + sec = self.openstack_connection.get_security_group(name_or_id=openstack_id) + if sec: + logger.info(f"Security group [{openstack_id}] already exists.") + return sec["id"] + logger.info(f"No security Group for [{openstack_id}] exists. Creating.. ") + new_security_group = self.openstack_connection.create_security_group( + name=openstack_id, description=f"VM ID: {openstack_id} Security Group" + ) + return new_security_group["id"] + def get_or_create_project_security_group(self, project_name, project_id): security_group_name = f"{project_name}_{project_id}" logger.info( @@ -1040,9 +1115,9 @@ def delete_server(self, openstack_id: str) -> None: server=server, security_group=sec ) if ( - sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sec.name - and not self.is_security_group_in_use(security_group_id=sec.id) + sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME + and ("bibigrid" not in sec.name or "master" not in server.name) + and not self.is_security_group_in_use(security_group_id=sec.id) ): self.openstack_connection.delete_security_group(sg) self.openstack_connection.compute.delete_server(server.id, force=True) @@ -1079,40 +1154,51 @@ def get_vm_ports(self, openstack_id: str) -> dict[str, str]: return {"port": str(ssh_port), "udp": str(udp_port)} def create_userdata( - self, - volume_ids_path_new: list[dict[str, str]], - volume_ids_path_attach: list[dict[str, str]], - additional_keys: list[str], + self, + volume_ids_path_new: list[dict[str, str]], + volume_ids_path_attach: list[dict[str, str]], + additional_keys: list[str], ) -> str: - init_script = self.create_mount_init_script( - new_volumes=volume_ids_path_new, - attach_volumes=volume_ids_path_attach, + + unlock_ubuntu_user_script = "#!/bin/bash\npasswd -u ubuntu\n" + unlock_ubuntu_user_script_encoded = encodeutils.safe_encode( + unlock_ubuntu_user_script.encode("utf-8") ) + init_script = unlock_ubuntu_user_script_encoded + if additional_keys: - if init_script: - add_key_script = self.create_add_keys_script(keys=additional_keys) - init_script = ( + add_key_script = self.create_add_keys_script(keys=additional_keys) + init_script = ( add_key_script + encodeutils.safe_encode("\n".encode("utf-8")) + init_script - ) + ) + if volume_ids_path_new or volume_ids_path_attach: + mount_script = self.create_mount_init_script( + new_volumes=volume_ids_path_new, + attach_volumes=volume_ids_path_attach, + ) + init_script = ( + init_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + mount_script + + ) - else: - init_script = self.create_add_keys_script(keys=additional_keys) return init_script def start_server( - self, - flavor_name: str, - image_name: str, - servername: str, - metadata: dict[str, str], - public_key: str, - research_environment_metadata: Union[ResearchEnvironmentMetadata, None] = None, - volume_ids_path_new: Union[list[dict[str, str]], None] = None, - volume_ids_path_attach: Union[list[dict[str, str]], None] = None, - additional_keys: Union[list[str], None] = None, - additional_security_group_ids: Union[list[str], None] = None, + self, + flavor_name: str, + image_name: str, + servername: str, + metadata: dict[str, str], + public_key: str, + research_environment_metadata: Union[ResearchEnvironmentMetadata, None] = None, + volume_ids_path_new: Union[list[dict[str, str]], None] = None, + volume_ids_path_attach: Union[list[dict[str, str]], None] = None, + additional_keys: Union[list[str], None] = None, + additional_security_group_ids: Union[list[str], None] = None, ) -> str: logger.info(f"Start Server {servername}") @@ -1123,7 +1209,7 @@ def start_server( network: Network = self.get_network() key_name = f"{servername}_{metadata['project_name']}" logger.info(f"Key name {key_name}") - security_groups = self.DEFAULT_SECURITY_GROUPS + security_groups = self._get_default_security_groups() if research_environment_metadata: security_groups.append( self.get_or_create_research_environment_security_group( @@ -1194,20 +1280,20 @@ def start_server( raise DefaultException(message=str(e)) def start_server_with_playbook( - self, - flavor_name: str, - image_name: str, - servername: str, - metadata: dict[str, str], - research_environment_metadata: ResearchEnvironmentMetadata, - volume_ids_path_new: list[dict[str, str]] = None, # type: ignore - volume_ids_path_attach: list[dict[str, str]] = None, # type: ignore - additional_keys: list[str] = None, # type: ignore - additional_security_group_ids=None, # type: ignore + self, + flavor_name: str, + image_name: str, + servername: str, + metadata: dict[str, str], + research_environment_metadata: ResearchEnvironmentMetadata, + volume_ids_path_new: list[dict[str, str]] = None, # type: ignore + volume_ids_path_attach: list[dict[str, str]] = None, # type: ignore + additional_keys: list[str] = None, # type: ignore + additional_security_group_ids=None, # type: ignore ) -> tuple[str, str]: logger.info(f"Start Server {servername}") - security_groups = self.DEFAULT_SECURITY_GROUPS + security_groups = self._get_default_security_groups() if research_environment_metadata: security_groups.append( self.get_or_create_research_environment_security_group( @@ -1339,16 +1425,16 @@ def add_udp_security_group(self, server_id): return def add_cluster_machine( - self, - cluster_id: str, - cluster_user: str, - cluster_group_id: list[str], - image_name: str, - flavor_name: str, - name: str, - key_name: str, - batch_idx: int, - worker_idx: int, + self, + cluster_id: str, + cluster_user: str, + cluster_group_id: list[str], + image_name: str, + flavor_name: str, + name: str, + key_name: str, + batch_idx: int, + worker_idx: int, ) -> str: logger.info(f"Add machine to {cluster_id}") image: Image = self.get_image(name_or_id=image_name, replace_inactive=True) diff --git a/simple_vm_client/openstack_connector/scripts/bash/mount.sh b/simple_vm_client/openstack_connector/scripts/bash/mount.sh index 685c53c..dc23ac0 100644 --- a/simple_vm_client/openstack_connector/scripts/bash/mount.sh +++ b/simple_vm_client/openstack_connector/scripts/bash/mount.sh @@ -1,5 +1,4 @@ #!/bin/bash -sudo touch test declare -a volumes_new=VOLUME_IDS_NEW declare -a paths_new=VOLUME_PATHS_NEW declare -a volumes_attach=VOLUME_IDS_ATTACH diff --git a/simple_vm_client/ttypes.py b/simple_vm_client/ttypes.py index e24da55..d1230a8 100644 --- a/simple_vm_client/ttypes.py +++ b/simple_vm_client/ttypes.py @@ -2665,6 +2665,123 @@ def __ne__(self, other): return not (self == other) +class SecurityGroupRuleNotFoundException(TException): + """ + Attributes: + - message: Server not found. + - name_or_id + + """ + + def __init__( + self, + message=None, + name_or_id=None, + ): + super(SecurityGroupRuleNotFoundException, self).__setattr__("message", message) + super(SecurityGroupRuleNotFoundException, self).__setattr__( + "name_or_id", name_or_id + ) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash( + ( + self.message, + self.name_or_id, + ) + ) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + name_or_id = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + name_or_id = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + name_or_id=name_or_id, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write( + oprot._fast_encode(self, [self.__class__, self.thrift_spec]) + ) + return + oprot.writeStructBegin("SecurityGroupRuleNotFoundException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString( + self.message.encode("utf-8") + if sys.version_info[0] == 2 + else self.message + ) + oprot.writeFieldEnd() + if self.name_or_id is not None: + oprot.writeFieldBegin("name_or_id", TType.STRING, 2) + oprot.writeString( + self.name_or_id.encode("utf-8") + if sys.version_info[0] == 2 + else self.name_or_id + ) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["%s=%r" % (key, value) for key, value in self.__dict__.items()] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class FlavorNotFoundException(TException): """ Attributes: @@ -4342,6 +4459,24 @@ def __ne__(self, other): None, ), # 2 ) +all_structs.append(SecurityGroupRuleNotFoundException) +SecurityGroupRuleNotFoundException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "name_or_id", + "UTF8", + None, + ), # 2 +) all_structs.append(FlavorNotFoundException) FlavorNotFoundException.thrift_spec = ( None, # 0 diff --git a/simple_vm_client/util/state_enums.py b/simple_vm_client/util/state_enums.py index c4bca86..9554c82 100644 --- a/simple_vm_client/util/state_enums.py +++ b/simple_vm_client/util/state_enums.py @@ -69,6 +69,7 @@ class VmStates(Enum): ERROR = "error" SHELVED = "shelved" SHELVED_OFFLOADED = "shelved_offloaded" + DISABLED = "disabled" # Custom vm states NOT_FOUND = "not_found"