From 35265872634870af6e7ed57038e154523cab87c3 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier <36056823+XaverStiensmeier@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:06:45 +0100 Subject: [PATCH 1/2] Create dependabot.yml (#479) --- .github/dependabot.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..e644f86c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + open-pull-requests-limit: 10 + versioning-strategy: "widen" + target: + versions: [">=3.0.0"] From 9b0f63281a3983ba3e3c299f439a97958ca8ea51 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier <36056823+XaverStiensmeier@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:09:12 +0100 Subject: [PATCH 2/2] Code cleanup and minor improvement (#482) * fixed :param and :return to @param and @return * many spelling mistakes fixed * added bibigrid_version to common configuration --- bibigrid/core/actions/check.py | 8 +- bibigrid/core/actions/create.py | 54 ++++---- bibigrid/core/actions/ide.py | 2 +- bibigrid/core/actions/list_clusters.py | 48 +++---- bibigrid/core/actions/terminate.py | 3 +- bibigrid/core/actions/update.py | 12 +- bibigrid/core/provider.py | 124 +++++++++--------- bibigrid/core/startup.py | 25 ++-- bibigrid/core/utility/ansible_commands.py | 14 +- bibigrid/core/utility/ansible_configurator.py | 72 +++++----- .../core/utility/command_line_interpreter.py | 2 +- .../utility/handler/configuration_handler.py | 26 ++-- .../core/utility/handler/provider_handler.py | 24 ++-- bibigrid/core/utility/handler/ssh_handler.py | 78 +++++------ bibigrid/core/utility/id_generation.py | 14 +- .../utility/paths/ansible_resources_path.py | 4 +- bibigrid/core/utility/paths/bin_path.py | 4 +- .../core/utility/validate_configuration.py | 76 ++++++----- .../core/utility/wireguard/wireguard_keys.py | 6 +- bibigrid/models/return_threading.py | 6 +- bibigrid/openstack/openstack_provider.py | 52 ++++---- requirements.txt | 20 ++- tests/test_id_generation.py | 2 +- 23 files changed, 344 insertions(+), 332 deletions(-) diff --git a/bibigrid/core/actions/check.py b/bibigrid/core/actions/check.py index 0ad37c7b..214021a8 100644 --- a/bibigrid/core/actions/check.py +++ b/bibigrid/core/actions/check.py @@ -7,10 +7,10 @@ def check(configurations, providers, log): """ Uses validate_configuration to validate given configuration. - :param configurations: list of configurations (dicts) - :param providers: list of providers - :param log: - :return: + @param configurations: list of configurations (dicts) + @param providers: list of providers + @param log: + @return: """ success = validate_configuration.ValidateConfiguration(configurations, providers, log).validate() check_result = "succeeded! Cluster is ready to start." if success else "failed!" diff --git a/bibigrid/core/actions/create.py b/bibigrid/core/actions/create.py index 6f43920a..966ec402 100644 --- a/bibigrid/core/actions/create.py +++ b/bibigrid/core/actions/create.py @@ -17,8 +17,8 @@ from bibigrid.core.utility import id_generation from bibigrid.core.utility import image_selection from bibigrid.core.utility.handler import ssh_handler -from bibigrid.core.utility.paths import ansible_resources_path as aRP -from bibigrid.core.utility.paths import bin_path as biRP +from bibigrid.core.utility.paths import ansible_resources_path as a_rp +from bibigrid.core.utility.paths import bin_path from bibigrid.models import exceptions from bibigrid.models import return_threading from bibigrid.models.exceptions import ExecutionException, ConfigurationException @@ -26,7 +26,7 @@ PREFIX = "bibigrid" SEPARATOR = "-" PREFIX_WITH_SEP = PREFIX + SEPARATOR -FILEPATHS = [(aRP.PLAYBOOK_PATH, aRP.PLAYBOOK_PATH_REMOTE), (biRP.BIN_PATH, biRP.BIN_PATH_REMOTE)] +FILEPATHS = [(a_rp.PLAYBOOK_PATH, a_rp.PLAYBOOK_PATH_REMOTE), (bin_path.BIN_PATH, bin_path.BIN_PATH_REMOTE)] def get_identifier(identifier, cluster_id, additional=""): @@ -67,10 +67,10 @@ def __init__(self, providers, configurations, config_path, log, debug=False, clu """ Additionally sets (unique) cluster_id, public_key_commands (to copy public keys to master) and key_name. Call create() to actually start server. - :param providers: List of providers (provider) - :param configurations: List of configurations (dict) - :param config_path: string that is the path to config-file - :param debug: Bool. If True Cluster offer shut-down after create and + @param providers: List of providers (provider) + @param configurations: List of configurations (dict) + @param config_path: string that is the path to config-file + @param debug: Bool. If True Cluster offer shut-down after create and will ask before shutting down on errors """ self.log = log @@ -104,7 +104,7 @@ def generate_keypair(self): See here for why using python module ECDSA wasn't successful https://stackoverflow.com/questions/71194770/why-does-creating-ecdsa-keypairs-via-python-differ-from-ssh -keygen-t-ecdsa-and - :return: + @return: """ self.log.info("Generating keypair") # create KEY_FOLDER if it doesn't exist @@ -129,7 +129,7 @@ def generate_security_groups(self): """ Generate a security groups: - default with basic rules for the cluster - - wireguard when more than one provider is used (= multicloud) + - wireguard when more than one provider is used (= multi-cloud) """ self.log.info("Generating Security Groups") for provider, configuration in zip(self.providers, self.configurations): @@ -163,9 +163,9 @@ def generate_security_groups(self): def start_vpn_or_master_instance(self, configuration, provider): """ Start master/vpn-worker of a provider - :param configuration: dict configuration of said provider - :param provider: provider - :return: + @param configuration: dict configuration of said provider. + @param provider: provider + @return: """ identifier, instance_type, volumes = self.prepare_vpn_or_master_args(configuration, provider) external_network = provider.get_external_network(configuration["network"]) @@ -209,9 +209,9 @@ def start_vpn_or_master_instance(self, configuration, provider): def prepare_vpn_or_master_args(self, configuration, provider): """ Prepares start_instance arguments for master/vpn - :param configuration: configuration (dict) of said master/vpn - :param provider: provider - :return: arguments needed by start_instance + @param configuration: configuration (dict) of said master/vpn + @param provider: provider + @return: arguments needed by start_instance """ if configuration.get("masterInstance"): instance_type = configuration["masterInstance"] @@ -247,9 +247,9 @@ def initialize_instances(self): def prepare_volumes(self, provider, mounts): """ Creates volumes from snapshots and returns all volumes (pre-existing and newly created) - :param provider: provider on which the volumes and snapshots exist - :param mounts: volumes or snapshots - :return: list of pre-existing and newly created volumes + @param provider: provider on which the volumes and snapshots exist + @param mounts: volumes or snapshots + @return: list of pre-existing and newly created volumes """ if mounts: self.log.info("Preparing volumes") @@ -275,7 +275,7 @@ def prepare_configurations(self): """ Makes sure that subnet and network key are set for each configuration. If none is set a keyError will be raised and caught in create. - :return: + @return: """ for configuration, provider in zip(self.configurations, self.providers): configuration["cloud_identifier"] = provider.cloud_specification["identifier"] @@ -298,15 +298,15 @@ def prepare_configurations(self): def upload_data(self): """ Configures ansible and then uploads the modified files and all necessary data to the master - :return: + @return: """ self.log.debug("Uploading ansible Data") - for folder in [aRP.VARS_FOLDER, aRP.GROUP_VARS_FOLDER, aRP.HOST_VARS_FOLDER]: + for folder in [a_rp.VARS_FOLDER, a_rp.GROUP_VARS_FOLDER, a_rp.HOST_VARS_FOLDER]: if not os.path.isdir(folder): self.log.info("%s not found. Creating folder.", folder) os.mkdir(folder) - if not os.path.isfile(aRP.HOSTS_FILE): - with open(aRP.HOSTS_FILE, 'a', encoding='utf-8') as hosts_file: + if not os.path.isfile(a_rp.HOSTS_FILE): + with open(a_rp.HOSTS_FILE, 'a', encoding='utf-8') as hosts_file: hosts_file.write("# placeholder file for worker DNS entries (see 003-dns)") ansible_configurator.configure_ansible_yaml(providers=self.providers, configurations=self.configurations, @@ -323,7 +323,7 @@ def upload_data(self): def start_start_instance_threads(self): """ Starts for each provider a start_instances thread and joins them. - :return: + @return: """ start_instance_threads = [] for configuration, provider in zip(self.configurations, self.providers): @@ -337,7 +337,7 @@ def start_start_instance_threads(self): def extended_network_configuration(self): """ Configure master/vpn-worker network for a multi/hybrid cloud - :return: + @return: """ if len(self.providers) == 1: return @@ -368,7 +368,7 @@ def create(self): # pylint: disable=too-many-branches,too-many-statements """ Creates cluster and logs helpful cluster-info afterwards. If debug is set True it offers termination after starting the cluster. - :return: exit_state + @return: exit_state """ try: self.generate_keypair() @@ -433,7 +433,7 @@ def log_cluster_start_info(self): SSH: How to connect to master via SSH Terminate: What bibigrid command is needed to terminate the created cluster Detailed cluster info: How to log detailed info about the created cluster - :return: + @return: """ gateway = self.configurations[0].get("gateway") ssh_ip = self.master_ip diff --git a/bibigrid/core/actions/ide.py b/bibigrid/core/actions/ide.py index 8f96cbd3..a9b26880 100644 --- a/bibigrid/core/actions/ide.py +++ b/bibigrid/core/actions/ide.py @@ -40,7 +40,7 @@ def sigint_handler(caught_signal, frame): # pylint: disable=unused-argument def is_used(ip_address): """ https://stackoverflow.com/questions/62000168/how-to-check-if-ssh-tunnel-is-being-used - :return: + @return: """ ports_used = [] with subprocess.Popen(["netstat", "-na"], stdout=subprocess.PIPE) as process: diff --git a/bibigrid/core/actions/list_clusters.py b/bibigrid/core/actions/list_clusters.py index 90e7aa5e..e6e3cf03 100644 --- a/bibigrid/core/actions/list_clusters.py +++ b/bibigrid/core/actions/list_clusters.py @@ -14,9 +14,9 @@ def dict_clusters(providers, log): """ Creates a dictionary containing all servers by type and provider information - :param providers: list of all providers - :param log: - :return: list of all clusters in yaml format + @param providers: list of all providers + @param log: + @return: list of all clusters in yaml format """ log.info("Creating cluster dictionary...") cluster_dict = {} @@ -39,11 +39,11 @@ def setup(cluster_dict, cluster_id, server, provider): """ Determines cluster_id. Generates empty entry for cluster_id in cluster_dict. - :param server: found server (dict) - :param cluster_id: id of said cluster - :param cluster_dict: dict containing all found servers by their cluster_id - :param provider: server's provider - :return: cluster_id + @param server: found server (dict) + @param cluster_id: id of said cluster + @param cluster_dict: dict containing all found servers by their cluster_id + @param provider: server's provider + @return: cluster_id """ if not cluster_dict.get(cluster_id): cluster_dict[cluster_id] = {} @@ -57,10 +57,10 @@ def log_list(cluster_id, providers, log): """ Calls dict_clusters and gives a visual representation of the found cluster. Detail depends on whether a cluster_id is given or not. - :param cluster_id: - :param providers: - :param log: - :return: + @param cluster_id: + @param providers: + @param log: + @return: """ cluster_dict = dict_clusters(providers=providers, log=log) if cluster_id: # pylint: disable=too-many-nested-blocks @@ -68,7 +68,7 @@ def log_list(cluster_id, providers, log): log.info("Printing specific cluster_dictionary") master_count, worker_count, vpn_count = get_size_overview(cluster_dict[cluster_id], log) log.log(42, f"\tCluster has {master_count} master, {vpn_count} vpngtw and {worker_count} regular workers. " - f"The cluster is spread over {vpn_count + master_count} reachable provider(s).") + f"The cluster is spread over {vpn_count + master_count} reachable provider(s).") log.log(42, pprint.pformat(cluster_dict[cluster_id])) else: log.info("Cluster with cluster-id {cluster_id} not found.") @@ -101,9 +101,9 @@ def log_list(cluster_id, providers, log): def get_size_overview(cluster_dict, log): """ - :param cluster_dict: dictionary of cluster to size_overview - :param log: - :return: number of masters, number of workers, number of vpns + @param cluster_dict: dictionary of cluster to size_overview + @param log: + @return: number of masters, number of workers, number of vpns """ log.info("Printing size overview") master_count = int(bool(cluster_dict.get("master"))) @@ -115,8 +115,8 @@ def get_size_overview(cluster_dict, log): def get_networks(cluster_dict): """ Gets all addresses of servers - :param cluster_dict: dictionary of clusters to find addresses - :return: dict containing addresses + @param cluster_dict: dictionary of clusters to find addresses + @return: dict containing addresses """ master = cluster_dict["master"] addresses = [{master["provider"]: list(master["addresses"].keys())}] @@ -128,8 +128,8 @@ def get_networks(cluster_dict): def get_security_groups(cluster_dict): """ Gets all security group of servers - :param cluster_dict: dictionary of clusters to find security_groups - :return: dict containing security_groups + @param cluster_dict: dictionary of clusters to find security_groups + @return: dict containing security_groups """ master = cluster_dict["master"] security_groups = [{master["provider"]: master["security_groups"]}] @@ -141,10 +141,10 @@ def get_security_groups(cluster_dict): def get_master_access_ip(cluster_id, master_provider, log): """ Returns master's ip of cluster cluster_id - :param master_provider: master's provider - :param cluster_id: Id of cluster - :param log: - :return: public ip of master + @param master_provider: master's provider + @param cluster_id: id of cluster + @param log: + @return: public ip of master """ log.info("Finding master ip for cluster %s...", cluster_id) servers = master_provider.list_servers() diff --git a/bibigrid/core/actions/terminate.py b/bibigrid/core/actions/terminate.py index 15a5a859..44c480ae 100644 --- a/bibigrid/core/actions/terminate.py +++ b/bibigrid/core/actions/terminate.py @@ -57,6 +57,7 @@ def terminate_servers(server_list, cluster_id, provider, log): @param server_list: list of server dicts. All servers are from provider @param cluster_id: id of cluster to terminate @param provider: provider that holds all servers in server_list + @param log: @return: a list of the servers' (that were to be terminated) termination states """ log.info("Deleting servers on provider %s...", provider.cloud_specification['identifier']) @@ -73,7 +74,7 @@ def terminate_servers(server_list, cluster_id, provider, log): def terminate_server(provider, server, log): """ Terminates a single server and stores the termination state - @param provider: the provider that holds the server + @param provider: the provider that holds the server. @param server: the server that is to be terminated @param log: @return: true if the server has been terminated, false else diff --git a/bibigrid/core/actions/update.py b/bibigrid/core/actions/update.py index ed866a12..efe9aaf7 100644 --- a/bibigrid/core/actions/update.py +++ b/bibigrid/core/actions/update.py @@ -2,10 +2,10 @@ Module that contains methods to update the master playbook """ -from bibigrid.core.utility import ansible_commands as aC +from bibigrid.core.utility import ansible_commands as a_c from bibigrid.core.utility.handler import ssh_handler -from bibigrid.core.utility.paths import ansible_resources_path as aRP -from bibigrid.core.utility.paths import bin_path as biRP +from bibigrid.core.utility.paths import ansible_resources_path as a_rp +from bibigrid.core.utility.paths import bin_path from bibigrid.core.utility.handler import cluster_ssh_handler @@ -18,8 +18,8 @@ def update(cluster_id, master_provider, master_configuration, log): ssh_handler.execute_ssh(floating_ip=master_ip, private_key=used_private_key, username=ssh_user, log=log, gateway=master_configuration.get("gateway", {}), - commands=[aC.EXECUTE], - filepaths=[(aRP.PLAYBOOK_PATH, aRP.PLAYBOOK_PATH_REMOTE), - (biRP.BIN_PATH, biRP.BIN_PATH_REMOTE)]) + commands=[a_c.EXECUTE], + filepaths=[(a_rp.PLAYBOOK_PATH, a_rp.PLAYBOOK_PATH_REMOTE), + (bin_path.BIN_PATH, bin_path.BIN_PATH_REMOTE)]) return 0 return 1 diff --git a/bibigrid/core/provider.py b/bibigrid/core/provider.py index 133088cb..61cc5012 100644 --- a/bibigrid/core/provider.py +++ b/bibigrid/core/provider.py @@ -27,8 +27,8 @@ def __init__(self, cloud_specification): def create_application_credential(self, name=None): """ Creates an application credential with name name - :param name: Name of new application credential - :return: the application credential dictionary + @param name: Name of new application credential + @return: the application credential dictionary """ @abstractmethod @@ -36,55 +36,55 @@ def delete_application_credential_by_id_or_name(self, ac_id_or_name): """ Deletes existing application credential by id or name and returns true. If application credential not found it returns false. - :param ac_id_or_name: application credential id or name - :return: True if deleted else false + @param ac_id_or_name: application credential id or name + @return: True if deleted else false """ @abstractmethod def get_image_by_id_or_name(self, image_id_or_name): """ Returns image that has id or name image_id_or_name - :param image_id_or_name: identifier - :return: said image (dict) or none if not found + @param image_id_or_name: identifier + @return: said image (dict) or none if not found """ @abstractmethod def get_flavor(self, instance_type): """ Returns flavor that has id or name flavor_id_or_name - :param instance_type: identifier - :return: said flavor (dict) or none if not found + @param instance_type: identifier + @return: said flavor (dict) or none if not found """ @abstractmethod def get_volume_snapshot_by_id_or_name(self, snapshot_id_or_name): """ Returns snapshot that has id or name snapshot_id_or_name - :param snapshot_id_or_name: identifier - :return: said snapshot (dict) or none if not found + @param snapshot_id_or_name: identifier + @return: said snapshot (dict) or none if not found """ @abstractmethod def get_network_by_id_or_name(self, network_id_or_name): """ Returns network that has id or name network_id_or_name - :param network_id_or_name: identifier - :return: said network (dict) or none if not found + @param network_id_or_name: identifier + @return: said network (dict) or none if not found """ @abstractmethod def get_subnet_by_id_or_name(self, subnet_id_or_name): """ Returns subnet that has id or name subnet_id_or_name - :param subnet_id_or_name: identifier - :return: said subnet (dict) or none if not found + @param subnet_id_or_name: identifier + @return: said subnet (dict) or none if not found """ @abstractmethod def list_servers(self): """ Returns a list of all servers on logged in provider - :return: said list of servers or empty list if none found + @return: said list of servers or empty list if none found """ @abstractmethod @@ -93,112 +93,112 @@ def create_server(self, name, flavor, image, network, key_name=None, wait=True, """ Creates a new server and waits for it to be accessible if wait=True. If volumes are given, they are attached. Returns said server (dict) - :param name: name (str) - :param flavor: flavor/type (str) - :param image: image/bootable-medium (str) - :param network: network (str) - :param key_name: (str) - :param wait: (bool) - :param volumes: List of volumes (list (str)) - :param security_groups: List of security_groups list (str) - :return: server (dict) + @param name: name (str) + @param flavor: flavor/type (str) + @param image: image/bootable-medium (str) + @param network: network (str) + @param key_name: (str) + @param wait: (bool) + @param volumes: List of volumes (list (str)) + @param security_groups: List of security_groups list (str) + @return: server (dict) """ @abstractmethod def delete_server(self, name_or_id, delete_ips=True): """ Deletes server and floating_ip as well if delete_ips is true. The resource is then free again - :param name_or_id: - :param delete_ips: - :return: True if delete succeeded, False otherwise + @param name_or_id: + @param delete_ips: + @return: True if delete succeeded, False otherwise """ @abstractmethod def delete_keypair(self, key_name): """ Deletes keypair with key_name - :param key_name: (str) - :return: True if delete succeeded, False otherwise + @param key_name: (str) + @return: True if delete succeeded, False otherwise """ @abstractmethod def get_server_group_by_id_or_name(self, server_group_id_or_name): """ Returns server_group that has id or name server_group_id_or_name - :param server_group_id_or_name: identifier - :return: said server_group (dict) or none if not found + @param server_group_id_or_name: identifier + @return: said server_group (dict) or none if not found """ @abstractmethod def close(self): """ Closes connection - :return: + @return: """ @abstractmethod def create_keypair(self, name, public_key): """ - Creates a new keypair with name name and public_key public_key - :param name: name of new keypair - :param public_key: public_key of new keypair - :return: + Creates a new keypair with name and public_key + @param name: name of new keypair + @param public_key: public_key of new keypair + @return: """ @abstractmethod def get_network_id_by_subnet(self, subnet): """ Gets network_id by subnet - :param subnet: id (str) - :return: (str) + @param subnet: id (str) + @return: (str) """ @abstractmethod def get_subnet_ids_by_network(self, network): """ Gets subnet_ids (list (str)) by network_id - :param network: id (str) - :return: subnet_ids (list (str)) + @param network: id (str) + @return: subnet_ids (list (str)) """ @abstractmethod def get_free_resources(self): """ Gets free resources. If a resource cannot be determined, assume maximum is free. - :return: Dictionary containing the free resources + @return: Dictionary containing the free resources """ @abstractmethod def get_volume_by_id_or_name(self, name_or_id): """ Returns volume that has id or name name_or_id - :param name_or_id: identifier - :return: said volume (dict) or none if not found + @param name_or_id: identifier + @return: said volume (dict) or none if not found """ @abstractmethod def create_volume_from_snapshot(self, snapshot_name_or_id): """ Creates a volume from snapshot. - :param snapshot_name_or_id: name or id of snapshot - :return: id of created volume or none if failed + @param snapshot_name_or_id: name or id of snapshot + @return: id of created volume or none if failed """ @abstractmethod def get_external_network(self, network_name_or_id): """ Finds router interface with network id equal to given network and by that the external network. - :param network_name_or_id: Name or id of network - :return: Corresponding external network + @param network_name_or_id: Name or id of network + @return: Corresponding external network """ @abstractmethod def attach_available_floating_ip(self, network=None, server=None): """ Get a floating IP from a network or a pool and attach it to the server - :param network: - :param server: - :return: + @param network: + @param server: + @return: """ @abstractmethod @@ -218,7 +218,7 @@ def get_flavors(self): def get_active_images(self): """ Return a list of active images. - :return: A list of active images. + @return: A list of active images. """ return [image["name"] for image in self.get_images() if image["status"].lower() == "active"] @@ -230,39 +230,39 @@ def get_active_flavors(self): def set_allowed_addresses(self, id_or_ip, allowed_address_pairs): """ Set allowed address (or CIDR) for the given network interface/port - :param id_or_ip: id or ipv4 ip-address of the port/interface - :param allowed_address_pairs: a list of allowed address pairs. For example: + @param id_or_ip: id or ipv4 ip-address of the port/interface + @param allowed_address_pairs: a list of allowed address pairs. For example: [{ "ip_address": "23.23.23.1", "mac_address": "fa:16:3e:c4:cd:3f" }] - :return: + @return: """ @abstractmethod def create_security_group(self, name, rules): """ Create a security group and add given rules - :param name: Name of the security group to be created - :param rules: List of firewall rules to be added - :return: id of created security group + @param name: Name of the security group to be created + @param rules: List of firewall rules to be added + @return: id of created security group """ @abstractmethod def delete_security_group(self, name_or_id): """ Delete a security group - :param name_or_id : Name or Id of the security group to be deleted - :return: True if delete succeeded, False otherwise. + @param name_or_id : Name or id of the security group to be deleted + @return: True if delete succeeded, False otherwise. """ @abstractmethod def append_rules_to_security_group(self, name_or_id, rules): """ Append firewall rules to given security group - :param name_or_id: - :param rules: - :return: + @param name_or_id: + @param rules: + @return: """ def get_mount_info_from_server(self, server): diff --git a/bibigrid/core/startup.py b/bibigrid/core/startup.py index 95b0c292..41aa75fb 100755 --- a/bibigrid/core/startup.py +++ b/bibigrid/core/startup.py @@ -40,8 +40,8 @@ def get_cluster_id_from_mem(): def set_logger_verbosity(verbosity): """ Sets verbosity, format and handler. - :param verbosity: level of verbosity - :return: + @param verbosity: level of verbosity + @return: """ capped_verbosity = min(verbosity, len(VERBOSITY_LIST) - 1) @@ -56,10 +56,10 @@ def set_logger_verbosity(verbosity): def run_action(args, configurations, config_path): """ Uses args to decide which action will be executed and executes said action. - :param args: command line arguments - :param configurations: list of configurations (dicts) - :param config_path: path to configurations-file - :return: + @param args: command line arguments + @param configurations: list of configurations (dicts) + @param config_path: path to configurations-file + @return: """ if args.version: LOG.info("Action version selected") @@ -79,13 +79,10 @@ def run_action(args, configurations, config_path): exit_state = check.check(configurations, providers, LOG) elif args.create: LOG.info("Action create selected") - creator = create.Create(providers=providers, - configurations=configurations, - log=LOG, - debug=args.debug, + creator = create.Create(providers=providers, configurations=configurations, log=LOG, debug=args.debug, config_path=config_path) LOG.log(42, "Creating a new cluster takes about 10 or more minutes depending on your cloud provider " - "and your configuration. Be patient.") + "and your configuration. Be patient.") exit_state = creator.create() else: if not args.cluster_id: @@ -95,9 +92,7 @@ def run_action(args, configurations, config_path): if args.cluster_id: if args.terminate: LOG.info("Action terminate selected") - exit_state = terminate.terminate(cluster_id=args.cluster_id, - providers=providers, - log=LOG, + exit_state = terminate.terminate(cluster_id=args.cluster_id, providers=providers, log=LOG, debug=args.debug) elif args.ide: LOG.info("Action ide selected") @@ -124,7 +119,7 @@ def run_action(args, configurations, config_path): def main(): """ Interprets command line, sets logger, reads configuration and runs selected action. Then exits. - :return: + @return: """ logging.basicConfig(format=LOGGER_FORMAT) # LOG.addHandler(logging.StreamHandler()) # stdout diff --git a/bibigrid/core/utility/ansible_commands.py b/bibigrid/core/utility/ansible_commands.py index fc6c2815..0a443e28 100644 --- a/bibigrid/core/utility/ansible_commands.py +++ b/bibigrid/core/utility/ansible_commands.py @@ -3,12 +3,12 @@ """ import os -import bibigrid.core.utility.paths.ansible_resources_path as aRP +import bibigrid.core.utility.paths.ansible_resources_path as a_rp -#TO_LOG = "| sudo tee -a /var/log/ansible.log" -#AIY = "apt-get -y install" -#SAU = "sudo apt-get update" -#NO_KEY_CHECK = "export ANSIBLE_HOST_KEY_CHECKING=False" +# TO_LOG = "| sudo tee -a /var/log/ansible.log" +# AIY = "apt-get -y install" +# SAU = "sudo apt-get update" +# NO_KEY_CHECK = "export ANSIBLE_HOST_KEY_CHECKING=False" NO_UPDATE = ("""sudo sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' """ """/etc/apt/apt.conf.d/20auto-upgrades""", "Disable apt auto update.") # Setup (Python for everyone) @@ -49,8 +49,8 @@ "Adjust playbook home permission.") MV_ANSIBLE_CONFIG = ( "sudo install -D /opt/playbook/ansible.cfg /etc/ansible/ansible.cfg", "Move ansible configuration.") -EXECUTE = (f"ansible-playbook {os.path.join(aRP.PLAYBOOK_PATH_REMOTE, aRP.SITE_YML)} -i " - f"{os.path.join(aRP.PLAYBOOK_PATH_REMOTE, aRP.ANSIBLE_HOSTS)} -l vpn", +EXECUTE = (f"ansible-playbook {os.path.join(a_rp.PLAYBOOK_PATH_REMOTE, a_rp.SITE_YML)} -i " + f"{os.path.join(a_rp.PLAYBOOK_PATH_REMOTE, a_rp.ANSIBLE_HOSTS)} -l vpn", "Execute ansible playbook. Be patient.") # ansible setup diff --git a/bibigrid/core/utility/ansible_configurator.py b/bibigrid/core/utility/ansible_configurator.py index 642e9ae4..0e7365a5 100644 --- a/bibigrid/core/utility/ansible_configurator.py +++ b/bibigrid/core/utility/ansible_configurator.py @@ -6,6 +6,7 @@ import mergedeep import yaml +from bibigrid.core.actions.version import __version__ from bibigrid.core.actions import create from bibigrid.core.actions import ide @@ -50,8 +51,8 @@ def generate_site_file_yaml(custom_roles): """ Generates site_yaml (dict). Deepcopy is used in case roles might differ between servers in the future. - :param custom_roles: ansibleRoles given by the config - :return: site_yaml (dict) + @param custom_roles: ansibleRoles given by the config + @return: site_yaml (dict) """ site_yaml = [{'hosts': 'master', "become": "yes", "vars_files": VARS_FILES, "roles": MASTER_ROLES}, {'hosts': 'vpngtw', "become": "yes", "vars_files": VARS_FILES, "roles": vpngtw_ROLES}, @@ -70,10 +71,10 @@ def write_host_and_group_vars(configurations, providers, cluster_id, log): # py """ ToDo filter what information really is necessary. Determined by further development Filters unnecessary information - :param configurations: configurations - :param providers: providers - :param cluster_id: To get proper naming - :return: filtered information (dict) + @param configurations: configurations + @param providers: providers + @param cluster_id: To get proper naming + @return: filtered information (dict) """ log.info("Generating instances file...") flavor_keys = ["name", "ram", "vcpus", "disk", "ephemeral"] @@ -156,18 +157,19 @@ def pass_through(dict_from, dict_to, key_from, key_to=None): def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_user, default_user, log): """ Generates common_configuration yaml (dict) - :param cidrs: str subnet cidrs (provider generated) - :param configurations: master configuration (first in file) - :param cluster_id: id of cluster - :param ssh_user: user for ssh connections - :param default_user: Given default user - :param log: - :return: common_configuration_yaml (dict) + @param cidrs: str subnet cidrs (provider generated) + @param configurations: master configuration (first in file) + @param cluster_id: id of cluster + @param ssh_user: user for ssh connections + @param default_user: Given default user + @param log: + @return: common_configuration_yaml (dict) """ master_configuration = configurations[0] log.info("Generating common configuration file...") # print(configuration.get("slurmConf", {})) - common_configuration_yaml = {"auto_mount": master_configuration.get("autoMount", False), "cluster_id": cluster_id, + common_configuration_yaml = {"bibigrid_version": __version__, + "auto_mount": master_configuration.get("autoMount", False), "cluster_id": cluster_id, "cluster_cidrs": cidrs, "default_user": default_user, "local_fs": master_configuration.get("localFS", False), "local_dns_lookup": master_configuration.get("localDNSlookup", False), @@ -211,11 +213,11 @@ def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_us def generate_ansible_hosts_yaml(ssh_user, configurations, cluster_id, log): # pylint: disable-msg=too-many-locals """ Generates ansible_hosts_yaml (inventory file). - :param ssh_user: str global SSH-username - :param configurations: dict - :param cluster_id: id of cluster - :param log: - :return: ansible_hosts yaml (dict) + @param ssh_user: str global SSH-username + @param configurations: dict + @param cluster_id: id of cluster + @param log: + @return: ansible_hosts yaml (dict) """ log.info("Generating ansible hosts file...") ansible_hosts_yaml = {"vpn": {"hosts": {}, @@ -249,9 +251,9 @@ def generate_ansible_hosts_yaml(ssh_user, configurations, cluster_id, log): # p def to_instance_host_dict(ssh_user, ip="localhost"): # pylint: disable=invalid-name """ Generates host entry - :param ssh_user: str global SSH-username - :param ip: str ip - :return: host entry (dict) + @param ssh_user: str global SSH-username + @param ip: str ip + @return: host entry (dict) """ host_yaml = {"ansible_connection": "ssh", "ansible_python_interpreter": PYTHON_INTERPRETER, "ansible_user": ssh_user} @@ -263,8 +265,8 @@ def to_instance_host_dict(ssh_user, ip="localhost"): # pylint: disable=invalid- def get_cidrs(configurations): """ Gets cidrs of all subnets in all providers - :param configurations: list of configurations (dict) - :return: + @param configurations: list of configurations (dict) + @return: """ all_cidrs = [] for configuration in configurations: @@ -297,9 +299,9 @@ def get_ansible_roles(ansible_roles, log): def get_ansible_galaxy_roles(ansible_galaxy_roles, log): """ Checks if ansible_galaxy_role have all necessary values and adds it to the return list if so. - :param ansible_galaxy_roles: - :param log: - :return: list of valid ansible_galaxy_roles + @param ansible_galaxy_roles: + @param log: + @return: list of valid ansible_galaxy_roles """ ansible_galaxy_roles_yaml = [] for ansible_galaxy_role in (ansible_galaxy_roles or []): @@ -317,9 +319,9 @@ def get_ansible_galaxy_roles(ansible_galaxy_roles, log): def generate_worker_specification_file_yaml(configurations, log): """ Generates worker_specification_file_yaml - :param configurations: list of configurations (dict) - :param log: - :return: worker_specification_yaml + @param configurations: list of configurations (dict) + @param log: + @return: worker_specification_yaml """ log.info("Generating worker specification file...") worker_groups_list = configuration_handler.get_list_by_key(configurations, "workerInstances", False) @@ -367,11 +369,11 @@ def add_wireguard_peers(configurations): def configure_ansible_yaml(providers, configurations, cluster_id, log): """ Generates and writes all ansible-configuration-yaml files. - :param providers: list of providers - :param configurations: list of configurations (dict) - :param cluster_id: id of cluster to create - :param log: - :return: + @param providers: list of providers + @param configurations: list of configurations (dict) + @param cluster_id: id of cluster to create + @param log: + @return: """ delete_old_vars(log) log.info("Writing ansible files...") diff --git a/bibigrid/core/utility/command_line_interpreter.py b/bibigrid/core/utility/command_line_interpreter.py index 30fed2c9..98ed1a35 100644 --- a/bibigrid/core/utility/command_line_interpreter.py +++ b/bibigrid/core/utility/command_line_interpreter.py @@ -28,7 +28,7 @@ def check_cid(cid): def interpret_command_line(): """ Interprets commandline. Used in startup.py - :return: + @return: """ parser = argparse.ArgumentParser(description='BiBiGrid easily sets up clusters within a cloud environment') parser.add_argument("-v", "--verbose", action="count", default=0, diff --git a/bibigrid/core/utility/handler/configuration_handler.py b/bibigrid/core/utility/handler/configuration_handler.py index c50a7f3d..bc124993 100644 --- a/bibigrid/core/utility/handler/configuration_handler.py +++ b/bibigrid/core/utility/handler/configuration_handler.py @@ -19,10 +19,10 @@ def read_configuration(log, path="bibigrid.yml", configuration_list=True): """ Reads yaml from file and returns configuration - :param log: - :param path: Path to yaml file - :param configuration_list: True if list is expected - :return: configurations (dict) + @param log: + @param path: Path to yaml file + @param configuration_list: True if list is expected + @return: configurations (dict) """ configuration = None if os.path.isfile(path): @@ -42,10 +42,10 @@ def read_configuration(log, path="bibigrid.yml", configuration_list=True): def get_list_by_key(configurations, key, get_empty=True): """ Returns a list of objects which are value to the key. - :param get_empty: if true empty configurations return None - :param configurations: YAML of configuration File containing the configuration-data for each provider - :param key: Key that is looked out for - :return: List of values of said key through all configs + @param get_empty: if true empty configurations return None + @param configurations: YAML of configuration File containing the configuration-data for each provider + @param key: Key that is looked out for + @return: List of values of said key through all configs """ return [configuration.get(key) for configuration in configurations if configuration.get(key) or get_empty] @@ -98,11 +98,11 @@ def get_clouds_files(log): def get_cloud_specification(cloud_name, clouds, clouds_public, log): """ As in openstack cloud_public_specification will be overwritten by cloud_private_specification - :param cloud_name: name of the cloud to look for in clouds.yaml - :param clouds: dict containing the data loaded from clouds.yaml - :param clouds_public: dict containing the data loaded from clouds-public.yaml - :param log: - :return: + @param cloud_name: name of the cloud to look for in clouds.yaml + @param clouds: dict containing the data loaded from clouds.yaml + @param clouds_public: dict containing the data loaded from clouds-public.yaml + @param log: + @return: """ cloud_full_specification = {} cloud_private_specification = clouds.get(cloud_name) diff --git a/bibigrid/core/utility/handler/provider_handler.py b/bibigrid/core/utility/handler/provider_handler.py index 3e74b1e9..338aba05 100644 --- a/bibigrid/core/utility/handler/provider_handler.py +++ b/bibigrid/core/utility/handler/provider_handler.py @@ -14,8 +14,9 @@ def get_provider_by_class_name(provider_name, """ Returns provider that is associated with the key provider_name in provider_dict. Otherwise, a KeyError is thrown. - :param provider_name: key of provider_dict - :return: provider + @param provider_name: key of provider_dict + @param provider_dict: + @return: provider """ return provider_dict[provider_name] @@ -23,9 +24,10 @@ def get_provider_by_class_name(provider_name, def get_provider_by_name(provider_name, provider_dict=PROVIDER_NAME_DICT): # pylint: disable=dangerous-default-value """ Returns provider that is associated with the key provider_name in provider_dict. - Otherwise a KeyError is thrown. - :param provider_name: key of provider_dict - :return: provider + Otherwise, a KeyError is thrown. + @param provider_name: key of provider_dict + @param provider_dict + @return: provider """ return provider_dict.get(provider_name) @@ -35,9 +37,9 @@ def get_provider_list_by_name_list(provider_name_list, cloud_specifications): Returns provider list for given provider_name_list If name is not found in PROVIDER_NAME_DICT, PROVIDER_CLASS_DICT is tried instead. If not found in both a key error is thrown. - :param provider_name_list: list of provider names - :param cloud_specifications: list of cloud specifications - :return: list of providers + @param provider_name_list: list of provider names + @param cloud_specifications: list of cloud specifications + @return: list of providers """ provider_list = [ (get_provider_by_name(provider_name) or get_provider_by_class_name(provider_name))(cloud_specification) for @@ -50,9 +52,9 @@ def get_providers(configurations, log): Reads list of provider_names from configurations. Determines list of providers by provider_names and returns it. If providers don't match a key error is thrown and the program exits with failure state 1. - :param configurations: - :param log: - :return: + @param configurations: + @param log: + @return: """ cloud_specifications = configuration_handler.get_cloud_specifications(configurations, log) if cloud_specifications: diff --git a/bibigrid/core/utility/handler/ssh_handler.py b/bibigrid/core/utility/handler/ssh_handler.py index f5c71ac3..ca6d51d9 100644 --- a/bibigrid/core/utility/handler/ssh_handler.py +++ b/bibigrid/core/utility/handler/ssh_handler.py @@ -1,6 +1,6 @@ """ This module handles ssh and sftp connections to master and vpngtw. It also holds general execution routines used to -setup the Cluster. +set up the Cluster. """ import os import socket @@ -10,15 +10,15 @@ import sympy import yaml -from bibigrid.core.utility import ansible_commands as aC +from bibigrid.core.utility import ansible_commands as a_c from bibigrid.models.exceptions import ConnectionException, ExecutionException PRIVATE_KEY_FILE = ".ssh/id_ecdsa" # to name bibigrid-temp keys identically on remote -ANSIBLE_SETUP = [aC.NO_UPDATE, aC.UPDATE, aC.PYTHON3_PIP, aC.ANSIBLE_PASSLIB, - (f"chmod 600 {PRIVATE_KEY_FILE}", "Adjust private key permissions."), aC.PLAYBOOK_HOME, - aC.PLAYBOOK_HOME_RIGHTS, aC.ADD_PLAYBOOK_TO_LINUX_HOME] +ANSIBLE_SETUP = [a_c.NO_UPDATE, a_c.UPDATE, a_c.PYTHON3_PIP, a_c.ANSIBLE_PASSLIB, + (f"chmod 600 {PRIVATE_KEY_FILE}", "Adjust private key permissions."), a_c.PLAYBOOK_HOME, + a_c.PLAYBOOK_HOME_RIGHTS, a_c.ADD_PLAYBOOK_TO_LINUX_HOME] # ANSIBLE_START = [aC.WAIT_READY, aC.UPDATE, aC.MV_ANSIBLE_CONFIG, aC.EXECUTE] # another UPDATE seems to not necessary. -ANSIBLE_START = [aC.WAIT_READY, aC.MV_ANSIBLE_CONFIG, aC.EXECUTE] +ANSIBLE_START = [a_c.WAIT_READY, a_c.MV_ANSIBLE_CONFIG, a_c.EXECUTE] VPN_SETUP = [("echo Example", "Echos an Example")] @@ -53,8 +53,8 @@ def get_ac_command(providers, name): def get_add_ssh_public_key_commands(ssh_public_key_files): """ Builds and returns the necessary commands to add given public keys to remote for additional access. - :param ssh_public_key_files: public keys to add - :return: list of public key add commands + @param ssh_public_key_files: public keys to add + @return: list of public key add commands """ commands = [] if ssh_public_key_files: @@ -69,11 +69,11 @@ def copy_to_server(sftp, local_path, remote_path, log): """ Recursively copies files and folders to server. If a folder is given as local_path, the structure within will be kept. - :param sftp: sftp connection - :param local_path: file or folder locally - :param remote_path: file or folder locally - :param log: - :return: + @param sftp: sftp connection + @param local_path: file or folder locally + @param remote_path: file or folder locally + @param log: + @return: """ log.debug("Copy %s to %s...", local_path, remote_path) if os.path.isfile(local_path): @@ -91,13 +91,13 @@ def is_active(client, floating_ip_address, private_key, username, log, gateway, """ Checks if connection is possible and therefore if server is active. Raises paramiko.ssh_exception.NoValidConnectionsError if timeout is reached - :param client: created client - :param floating_ip_address: ip to connect to - :param private_key: SSH-private_key - :param username: SSH-username - :param log: - :param timeout: how long to wait between ping - :param gateway: if node should be reached over a gateway port is set to 30000 + subnet * 256 + host + @param client: created client + @param floating_ip_address: ip to connect to + @param private_key: SSH-private_key + @param username: SSH-username + @param log: + @param timeout: how long to wait between ping + @param gateway: if node should be reached over a gateway port is set to 30000 + subnet * 256 + host (waiting grows quadratically till 2**timeout before accepting failure) """ attempts = 0 @@ -139,8 +139,8 @@ def line_buffered(f): """ https://stackoverflow.com/questions/25260088/paramiko-with-continuous-stdout temporary hangs? - :param f: - :return: + @param f: + @return: """ line_buf = b"" while not f.channel.exit_status_ready(): @@ -154,9 +154,9 @@ def line_buffered(f): def execute_ssh_cml_commands(client, commands, log): """ Executes commands and logs exit_status accordingly. - :param client: Client with connection to remote - :param commands: Commands to execute on remote - :param log: + @param client: Client with connection to remote + @param commands: Commands to execute on remote + @param log: """ for command in commands: _, ssh_stdout, _ = client.exec_command(command[0]) @@ -190,13 +190,13 @@ def ansible_preparation(floating_ip, private_key, username, log, gateway, comman Copies additional files and executes additional commands if given. The playbook is copied later, because it needs all servers setup and is not time intensive. See: create.update_playbooks - :param floating_ip: public ip of server to ansible-prepare - :param private_key: generated private key of all cluster-server - :param username: username of all server - :param log: - :param commands: additional commands to execute - :param filepaths: additional files to copy: (localpath, remotepath) - :param gateway + @param floating_ip: public ip of server to ansible-prepare + @param private_key: generated private key of all cluster-server + @param username: username of all server + @param log: + @param commands: additional commands to execute + @param filepaths: additional files to copy: (localpath, remotepath) + @param gateway """ if filepaths is None: filepaths = [] @@ -211,13 +211,13 @@ def ansible_preparation(floating_ip, private_key, username, log, gateway, comman def execute_ssh(floating_ip, private_key, username, log, gateway, commands=None, filepaths=None): """ Executes commands on remote and copies files given in filepaths - :param floating_ip: public ip of remote - :param private_key: key of remote - :param username: username of remote - :param commands: commands - :param log: - :param filepaths: filepaths (localpath, remotepath) - :param gateway: gateway if used + @param floating_ip: public ip of remote + @param private_key: key of remote + @param username: username of remote + @param commands: commands + @param log: + @param filepaths: filepaths (localpath, remotepath) + @param gateway: gateway if used """ if commands is None: commands = [] diff --git a/bibigrid/core/utility/id_generation.py b/bibigrid/core/utility/id_generation.py index 51ded9a9..f40336f1 100644 --- a/bibigrid/core/utility/id_generation.py +++ b/bibigrid/core/utility/id_generation.py @@ -13,7 +13,7 @@ def generate_cluster_id(): """ Generates an encrypted shortUUID with length MAX_ID_LENGTH - :return: + @return: """ uuid = shortuuid.ShortUUID() uuid.set_alphabet(CLUSTER_UUID_ALPHABET) @@ -23,8 +23,8 @@ def generate_cluster_id(): def generate_safe_cluster_id(providers): """ Generates a cluster_id and checks if cluster_id is not in use. When a unique id is found it is returned - :param providers: providers to check whether they use said cluster_id - :return: cluster_id + @param providers: providers to check whether they use said cluster_id + @return: cluster_id """ id_is_unique = False cluster_id = None @@ -37,9 +37,9 @@ def generate_safe_cluster_id(providers): def is_unique_cluster_id(cluster_id, providers): """ Checks if cluster_id is not in use on any provider - :param cluster_id: generated cluster_ird - :param providers: providers to check - :return: True if cluster_id is unique. False else. + @param cluster_id: generated cluster_ird + @param providers: providers to check + @return: True if cluster_id is unique. False else. """ for provider in providers: for server in provider.list_servers(): @@ -54,6 +54,6 @@ def is_unique_cluster_id(cluster_id, providers): def generate_munge_key(): """ Generates a munge key (UUID) for slurm - :return: + @return: """ return shortuuid.ShortUUID().random(32) diff --git a/bibigrid/core/utility/paths/ansible_resources_path.py b/bibigrid/core/utility/paths/ansible_resources_path.py index de4a99d5..34e81a74 100644 --- a/bibigrid/core/utility/paths/ansible_resources_path.py +++ b/bibigrid/core/utility/paths/ansible_resources_path.py @@ -4,7 +4,7 @@ import os -import bibigrid.core.utility.paths.basic_path as bP +import bibigrid.core.utility.paths.basic_path as b_p # UNIVERSAL ANSIBLE_HOSTS: str = "ansible_hosts" @@ -27,7 +27,7 @@ # LOCAL # ANSIBLE_CFG_PATH = os.path.join(bP.RESOURCES_PATH, ANSIBLE_CFG) PLAYBOOK = "playbook/" -PLAYBOOK_PATH: str = os.path.join(bP.RESOURCES_PATH, PLAYBOOK) +PLAYBOOK_PATH: str = os.path.join(b_p.RESOURCES_PATH, PLAYBOOK) HOSTS_FILE = os.path.join(PLAYBOOK_PATH, HOSTS_YML) HOSTS_CONFIG_FILE: str = os.path.join(PLAYBOOK_PATH, ANSIBLE_HOSTS) CONFIG_ROOT_PATH: str = os.path.join(PLAYBOOK_PATH, VARS_PATH) diff --git a/bibigrid/core/utility/paths/bin_path.py b/bibigrid/core/utility/paths/bin_path.py index 16b10011..344ac283 100644 --- a/bibigrid/core/utility/paths/bin_path.py +++ b/bibigrid/core/utility/paths/bin_path.py @@ -5,9 +5,9 @@ import os -import bibigrid.core.utility.paths.basic_path as bP +import bibigrid.core.utility.paths.basic_path as b_p BIN: str = "bin/" -BIN_PATH: str = os.path.join(bP.RESOURCES_PATH, BIN) +BIN_PATH: str = os.path.join(b_p.RESOURCES_PATH, BIN) BIN_PATH_REMOTE: str = BIN diff --git a/bibigrid/core/utility/validate_configuration.py b/bibigrid/core/utility/validate_configuration.py index cca9b113..1980abda 100644 --- a/bibigrid/core/utility/validate_configuration.py +++ b/bibigrid/core/utility/validate_configuration.py @@ -14,10 +14,10 @@ def evaluate(check_name, check_result, log): """ Logs check_result as warning if failed and as success if succeeded. - :param check_name: - :param check_result: - :param log: - :return: + @param check_name: + @param check_result: + @param log: + @return: """ if check_result: log.info("Checking %s: Success", check_name) @@ -30,10 +30,10 @@ def check_provider_data(provider_data_list, provider_count, log): """ Checks if all provider datas are unique and if enough providers are given #ToDo for multiple cloud locations additional provider data needs to be added - :param provider_data_list: list of all provider data - :param provider_count: number of providers - :param log: - :return: True if enough providers are given and all providers are unique + @param provider_data_list: list of all provider data + @param provider_count: number of providers + @param log: + @return: True if enough providers are given and all providers are unique """ log.info("Checking provider names") success = True @@ -89,12 +89,12 @@ def evaluate_ssh_public_key_file_security(ssh_public_key_file, log): def has_enough(maximum, needed, keeper, thing, log): """ Method logs and compares whether enough free things are available - :param maximum: maximum (available) resources of thing - :param needed: minimum needed to run - :param keeper: description of the object having the thing that is checked (for logging) - :param thing: description of what resource is checked (RAM for example) (for logging) - :param log: - :return: True if maximum is larger or equal to the needed + @param maximum: maximum (available) resources of thing + @param needed: minimum needed to run + @param keeper: description of the object having the thing that is checked (for logging) + @param thing: description of what resource is checked (RAM for example) (for logging) + @param log: + @return: True if maximum is larger or equal to the needed """ success = True if maximum >= needed: @@ -131,7 +131,7 @@ def check_clouds_yaml_security(log): def check_cloud_yaml(cloud_specification, log): """ - Check if cloud_specification is valid i.e. contains the necessary authentification data. + Check if cloud_specification is valid i.e. contains the necessary authentication data. @param cloud_specification: dict to check whether it is a valid cloud_specification @param log @return: True if cloud_specification is valid. False else. @@ -175,8 +175,8 @@ def __init__(self, configurations, providers, log): Sets configurations, providers and prepares the required_resources_dict. While executing the checks, needed resources are counted. In the end check_quotas will decide whether enough resources are available. - :param configurations: List of configurations (dicts) - :param providers: List of providers + @param configurations: List of configurations (dicts) + @param providers: List of providers """ self.log = log self.configurations = configurations @@ -194,12 +194,12 @@ def validate(self): The validation steps are as follows: Check connection can be established Check provider uniqueness - Check servergroup + Check server group Check instances are available Check images and volumes are available Check network and subnet are available Check quotas - :return: + @return: """ success = bool(self.providers) self.log.info("Validating config file...") @@ -223,7 +223,7 @@ def check_master_vpn_worker(self): Checks if first configuration has a masterInstance defined and every other configuration has a vpnInstance defined. If one is missing said provider wouldn't be reachable over the cluster, because no floating IP would be given. - :return: True if first configuration has a masterInstance and every other a vpnInstance + @return: True if first configuration has a masterInstance and every other a vpnInstance """ self.log.info("Checking master/vpn") success = True @@ -239,7 +239,7 @@ def check_master_vpn_worker(self): def check_provider_connections(self): """ Checks if all providers are reachable - :return: True if all providers are reachable + @return: True if all providers are reachable """ success = True providers_unconnectable = [] @@ -257,7 +257,7 @@ def check_provider_connections(self): def check_instances(self): """ Checks if all instances exist and image and instance-type are compatible - :return: true if image and instance-type (flavor) exist for all instances and are compatible + @return: true if image and instance-type (flavor) exist for all instances and are compatible """ self.log.info("Checking instance images and type") success = True @@ -280,10 +280,10 @@ def check_instances(self): def check_instance(self, instance_name, instance, provider): """ Checks if instance image exists and whether it is compatible with the defined instance/server type (flavor). - :param instance_name: containing name for logging purposes - :param instance: dict containing image, type and count (count is not used) - :param provider: provider - :return: true if type and image compatible and existing + @param instance_name: containing name for logging purposes + @param instance: dict containing image, type and count (count is not used) + @param provider: provider + @return: true if type and image compatible and existing """ self.required_resources_dict[provider.cloud_specification['identifier']]["instances"] += instance.get( "count") or 1 @@ -304,10 +304,10 @@ def check_instance(self, instance_name, instance, provider): def check_instance_type_image_combination(self, instance_type, instance_image, provider): """ Checks, if enough ram, disk space for instance_image are provided by instance_type on provider. - :param instance_type - :param instance_image - :param provider - :return true, if enough resources available + @param instance_type + @param instance_image + @param provider + @return true, if enough resources available """ success = True # check @@ -333,7 +333,7 @@ def check_instance_type_image_combination(self, instance_type, instance_image, p def check_volumes(self): """ Checking if volume or snapshot exists for all volumes - :return: True if all snapshot and volumes are found. Else false. + @return: True if all snapshot and volumes are found. Else false. """ self.log.info("Checking volumes...") success = True @@ -367,7 +367,7 @@ def check_volumes(self): def check_network(self): """ Check if network (or subnet) is accessible - :return True if any given network or subnet is accessible by provider + @return True if any given network or subnet is accessible by provider """ self.log.info("Checking network...") success = True @@ -391,11 +391,15 @@ def check_network(self): success = False else: self.log.info(f"Subnet '{subnet_name_or_id}' found") - return bool(success and (network_name_or_id or subnet_name_or_id)) + else: + self.log.warning( + f"Neither network nor subnet given in configuration {provider.cloud_specification['identifier']}.") + success = False + return success def check_server_group(self): """ - :return: True if server group accessible + @return: True if server group accessible """ success = True for configuration, provider in zip(self.configurations, self.providers): @@ -418,7 +422,7 @@ def check_quotas(self): Covered resources are: cores, floating_ips, instances, ram, volumes, volumeGigabytes, snapshots, backups and backupGigabytes. If a concrete provider implementation is unable to return remaining resources a maximum number is returned to make the check not fail because of the missing API implementation. - :return: True if check succeeded. Else false. + @return: True if check succeeded. Else false. """ self.log.info("Checking quotas") success = True @@ -433,7 +437,7 @@ def check_quotas(self): def check_ssh_public_key_files(self): """ Checks if keys listed in the config exist - :return: True if check succeeded. Else false. + @return: True if check succeeded. Else false. """ success = True for configuration in self.configurations: diff --git a/bibigrid/core/utility/wireguard/wireguard_keys.py b/bibigrid/core/utility/wireguard/wireguard_keys.py index c25c7316..2015f2e1 100755 --- a/bibigrid/core/utility/wireguard/wireguard_keys.py +++ b/bibigrid/core/utility/wireguard/wireguard_keys.py @@ -11,7 +11,7 @@ def generate(): """ Generates private and public key for wireguard - @return: tuple (privatekey_str, publickey_str) + @return: tuple (private_key_str, publickey_str) """ # generate private key private_key = X25519PrivateKey.generate() @@ -20,10 +20,10 @@ def generate(): format=serialization.PrivateFormat.Raw, encryption_algorithm=serialization.NoEncryption() ) - privatekey_str = codecs.encode(bytes_, 'base64').decode('utf8').strip() + private_key_str = codecs.encode(bytes_, 'base64').decode('utf8').strip() # derive public key publickey = private_key.public_key().public_bytes(encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw) publickey_str = codecs.encode(publickey, 'base64').decode('utf8').strip() - return privatekey_str, publickey_str + return private_key_str, publickey_str diff --git a/bibigrid/models/return_threading.py b/bibigrid/models/return_threading.py index a7c7a1b4..52c8b054 100644 --- a/bibigrid/models/return_threading.py +++ b/bibigrid/models/return_threading.py @@ -12,7 +12,9 @@ class ReturnThread(threading.Thread): - An exception occurred within the called function is raised by join() """ - def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): # pylint: disable=dangerous-default-value + # pylint: disable=dangerous-default-value + def __init__(self, group=None, target=None, name=None, args=(), + kwargs={}): threading.Thread.__init__(self, group, target, name, args, kwargs) self._return = None self._exc = None @@ -21,7 +23,7 @@ def run(self): if self._target is not None: try: self._return = self._target(*self._args, **self._kwargs) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except self._exc = exc def join(self, *args): diff --git a/bibigrid/openstack/openstack_provider.py b/bibigrid/openstack/openstack_provider.py index db0517e3..5efabf02 100644 --- a/bibigrid/openstack/openstack_provider.py +++ b/bibigrid/openstack/openstack_provider.py @@ -82,8 +82,8 @@ def delete_application_credential_by_id_or_name(self, ac_id_or_name): """ Deletes existing application credential by id or name and returns true. If application credential not found it returns false. - :param ac_id_or_name: application credential id or name - :return: True if deleted else false + @param ac_id_or_name: application credential id or name + @return: True if deleted else false """ try: self.keystone_client.application_credentials.delete(ac_id_or_name) # id @@ -137,9 +137,9 @@ def create_server(self, name, flavor, image, network, key_name=None, wait=True, def delete_server(self, name_or_id, delete_ips=True): """ Deletes server. floating_ip as well if delete_ips is true. The resources are then free again - :param name_or_id: - :param delete_ips: - :return: + @param name_or_id: + @param delete_ips: + @return: """ return self.conn.delete_server(name_or_id=name_or_id, wait=False, timeout=180, delete_ips=delete_ips, delete_ip_retry=1) @@ -154,7 +154,7 @@ def close(self): return self.conn.close() def create_keypair(self, name, public_key): - # When running a multicloud approach on the same provider and same account, + # When running a multi-cloud approach on the same provider and same account, # make sure that the keypair is only created ones. try: return self.conn.create_keypair(name=name, public_key=public_key) @@ -172,9 +172,9 @@ def get_subnet_ids_by_network(self, network): def get_free_resources(self): """ Uses openstack.block_storage to get all relevant volume resources. - Uses the openstack.compute to get all relevant compute resources. + Uses openstack compute to get all relevant compute resources. Floating-IP is not returned correctly by openstack. - :return: Dictionary containing the free resources + @return: Dictionary containing the free resources """ compute_limits = dict(self.conn.compute.get_limits()["absolute"]) volume_limits = dict(self.conn.block_storage.get_limits()["absolute"]) @@ -194,8 +194,8 @@ def create_volume_from_snapshot(self, snapshot_name_or_id): """ Uses the cinder API to create a volume from snapshot: https://github.com/openstack/python-cinderclient/blob/master/cinderclient/v3/volumes.py - :param snapshot_name_or_id: name or id of snapshot - :return: id of created volume + @param snapshot_name_or_id: name or id of snapshot + @return: id of created volume """ LOG.debug("Trying to create volume from snapshot") snapshot = self.conn.get_volume_snapshot(snapshot_name_or_id) @@ -217,8 +217,8 @@ def create_volume_from_snapshot(self, snapshot_name_or_id): def get_external_network(self, network_name_or_id): """ Finds router interface with network id equal to given network and by that the external network. - :param network_name_or_id:Name or id of network - :return:Corresponding external network + @param network_name_or_id:Name or id of network + @return:Corresponding external network """ network_id = self.conn.get_network(network_name_or_id)["id"] for router in self.conn.list_routers(): @@ -230,9 +230,9 @@ def get_external_network(self, network_name_or_id): def attach_available_floating_ip(self, network=None, server=None): """ Get a floating IP from a network or a pool and attach it to the server - :param network: - :param server: - :return: + @param network: + @param server: + @return: """ floating_ip = self.conn.available_floating_ip(network=network) if server: @@ -256,13 +256,13 @@ def get_flavors(self): def set_allowed_addresses(self, id_or_ip, allowed_address_pairs): """ Set allowed address (or CIDR) for the given network interface/port - :param id_or_ip: id or ip-address of the port/interfac - :param allowed_address: a list of allowed address pairs. For example: + @param id_or_ip: id or ip-address of the port/interface + @param allowed_address_pairs: a list of allowed address pairs. For example: [{ "ip_address": "23.23.23.1", "mac_address": "fa:16:3e:c4:cd:3f" }] - :return updated port: + @return updated port: """ # get port id if ip address is given if re.match(PATTERN_IPV4, id_or_ip): @@ -277,8 +277,8 @@ def set_allowed_addresses(self, id_or_ip, allowed_address_pairs): def create_security_group(self, name, rules=None): """ Create a security and add given rules - :param name: Name of the security group to be created - :param rules: List of firewall rules in the following format. + @param name: Name of the security group to be created + @param rules: List of firewall rules in the following format. rules = [{ "direction": "ingress" | "egress", "ethertype": "IPv4" | "IPv6", "protocol": "txp" | "udp" | "icmp" | None @@ -289,7 +289,7 @@ def create_security_group(self, name, rules=None): { ... } ] - :return: created security group + @return: created security group """ security_group = self.conn.create_security_group(name, f"Security group for {name}.") if rules is not None: @@ -299,8 +299,8 @@ def create_security_group(self, name, rules=None): def delete_security_group(self, name_or_id): """ Delete a security group - :param name_or_id : Name or Id of the security group to be deleted - :return: True if delete succeeded, False otherwise. + @param name_or_id : Name or id of the security group to be deleted + @return: True if delete succeeded, False otherwise. """ try: return self.conn.delete_security_group(name_or_id) @@ -310,9 +310,9 @@ def delete_security_group(self, name_or_id): def append_rules_to_security_group(self, name_or_id, rules): """ Append firewall rules to given security group - :param name_or_id: - :param rules: - :return: + @param name_or_id: + @param rules: + @return: """ for rule in rules: self.conn.create_security_group_rule(name_or_id, direction=rule["direction"], ethertype=rule["ethertype"], diff --git a/requirements.txt b/requirements.txt index d21a3828..c4ddcc7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,18 @@ openstacksdk==0.62 -mergedeep -paramiko +mergedeep~=1.3.4 +paramiko~=2.12.0 python-cinderclient python-keystoneclient python-novaclient python-openstackclient==6.0.0 -PyYAML -shortuuid -sshtunnel -sympy -seedir \ No newline at end of file +PyYAML~=6.0 +shortuuid~=1.0.11 +sshtunnel~=0.4.0 +sympy~=1.12 +seedir~=0.4.2 +cryptography~=38.0.4 +uvicorn~=0.23.2 +fastapi~=0.101.0 +pydantic~=2.1.1 +keystoneauth1~=5.1.0 +filelock~=3.13.1 \ No newline at end of file diff --git a/tests/test_id_generation.py b/tests/test_id_generation.py index 0e7c4009..64ae3703 100644 --- a/tests/test_id_generation.py +++ b/tests/test_id_generation.py @@ -16,7 +16,7 @@ class TestIDGeneration(TestCase): def test_generate_cluster_id(self): """ This test is not ideal, but prevents worst within a reasonable runtime - :return: + @return: """ test_list = [] for _ in range(10000):