Merge branch 'dev' into feature-increase-ssh-timeout

# Conflicts:
#	bibigrid/core/startup.py
#	bibigrid/core/utility/handler/ssh_handler.py

.github/dependabot.yml

@@ -0,0 +1,15 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 10
+    versioning-strategy: "widen"
+    target:
+      versions: [">=3.0.0"]

bibigrid/core/actions/check.py

@@ -7,10 +7,10 @@
 def check(configurations, providers, log):
     """
     Uses validate_configuration to validate given configuration.
-    :param configurations: list of configurations (dicts)
-    :param providers: list of providers
-    :param log:
-    :return:
+    @param configurations: list of configurations (dicts)
+    @param providers: list of providers
+    @param log:
+    @return:
     """
     success = validate_configuration.ValidateConfiguration(configurations, providers, log).validate()
     check_result = "succeeded! Cluster is ready to start." if success else "failed!"

bibigrid/core/actions/create.py

@@ -17,16 +17,16 @@
 from bibigrid.core.utility import id_generation
 from bibigrid.core.utility import image_selection
 from bibigrid.core.utility.handler import ssh_handler
-from bibigrid.core.utility.paths import ansible_resources_path as aRP
-from bibigrid.core.utility.paths import bin_path as biRP
+from bibigrid.core.utility.paths import ansible_resources_path as a_rp
+from bibigrid.core.utility.paths import bin_path
 from bibigrid.models import exceptions
 from bibigrid.models import return_threading
 from bibigrid.models.exceptions import ExecutionException, ConfigurationException
 
 PREFIX = "bibigrid"
 SEPARATOR = "-"
 PREFIX_WITH_SEP = PREFIX + SEPARATOR
-FILEPATHS = [(aRP.PLAYBOOK_PATH, aRP.PLAYBOOK_PATH_REMOTE), (biRP.BIN_PATH, biRP.BIN_PATH_REMOTE)]
+FILEPATHS = [(a_rp.PLAYBOOK_PATH, a_rp.PLAYBOOK_PATH_REMOTE), (bin_path.BIN_PATH, bin_path.BIN_PATH_REMOTE)]
 
 
 def get_identifier(identifier, cluster_id, additional=""):
@@ -67,10 +67,10 @@ def __init__(self, providers, configurations, config_path, log, debug=False, clu
         """
         Additionally sets (unique) cluster_id, public_key_commands (to copy public keys to master) and key_name.
         Call create() to actually start server.
-        :param providers: List of providers (provider)
-        :param configurations: List of configurations (dict)
-        :param config_path: string that is the path to config-file
-        :param debug: Bool. If True Cluster offer shut-down after create and
+        @param providers: List of providers (provider)
+        @param configurations: List of configurations (dict)
+        @param config_path: string that is the path to config-file
+        @param debug: Bool. If True Cluster offer shut-down after create and
         will ask before shutting down on errors
         """
         self.log = log
@@ -105,7 +105,7 @@ def generate_keypair(self):
         See here for why using python module ECDSA wasn't successful
         https://stackoverflow.com/questions/71194770/why-does-creating-ecdsa-keypairs-via-python-differ-from-ssh
         -keygen-t-ecdsa-and
-        :return:
+        @return:
         """
         self.log.info("Generating keypair")
         # create KEY_FOLDER if it doesn't exist
@@ -163,9 +163,9 @@ def generate_security_groups(self):
     def start_vpn_or_master_instance(self, configuration, provider):
         """
         Start master/vpn-worker of a provider
-        :param configuration: dict configuration of said provider
-        :param provider: provider
-        :return:
+        @param configuration: dict configuration of said provider.
+        @param provider: provider
+        @return:
         """
         identifier, instance_type, volumes = self.prepare_vpn_or_master_args(configuration, provider)
         external_network = provider.get_external_network(configuration["network"])
@@ -209,9 +209,9 @@ def start_vpn_or_master_instance(self, configuration, provider):
     def prepare_vpn_or_master_args(self, configuration, provider):
         """
         Prepares start_instance arguments for master/vpn
-        :param configuration: configuration (dict) of said master/vpn
-        :param provider: provider
-        :return: arguments needed by start_instance
+        @param configuration: configuration (dict) of said master/vpn
+        @param provider: provider
+        @return: arguments needed by start_instance
         """
         if configuration.get("masterInstance"):
             instance_type = configuration["masterInstance"]
@@ -247,9 +247,9 @@ def initialize_instances(self):
     def prepare_volumes(self, provider, mounts):
         """
         Creates volumes from snapshots and returns all volumes (pre-existing and newly created)
-        :param provider: provider on which the volumes and snapshots exist
-        :param mounts: volumes or snapshots
-        :return: list of pre-existing and newly created volumes
+        @param provider: provider on which the volumes and snapshots exist
+        @param mounts: volumes or snapshots
+        @return: list of pre-existing and newly created volumes
         """
         if mounts:
             self.log.info("Preparing volumes")
@@ -275,7 +275,7 @@ def prepare_configurations(self):
         """
         Makes sure that subnet and network key are set for each configuration.
         If none is set a keyError will be raised and caught in create.
-        :return:
+        @return:
         """
         for configuration, provider in zip(self.configurations, self.providers):
             configuration["cloud_identifier"] = provider.cloud_specification["identifier"]
@@ -298,15 +298,15 @@ def prepare_configurations(self):
     def upload_data(self):
         """
         Configures ansible and then uploads the modified files and all necessary data to the master
-        :return:
+        @return:
         """
         self.log.debug("Uploading ansible Data")
-        for folder in [aRP.VARS_FOLDER, aRP.GROUP_VARS_FOLDER, aRP.HOST_VARS_FOLDER]:
+        for folder in [a_rp.VARS_FOLDER, a_rp.GROUP_VARS_FOLDER, a_rp.HOST_VARS_FOLDER]:
             if not os.path.isdir(folder):
                 self.log.info("%s not found. Creating folder.", folder)
                 os.mkdir(folder)
-        if not os.path.isfile(aRP.HOSTS_FILE):
-            with open(aRP.HOSTS_FILE, 'a', encoding='utf-8') as hosts_file:
+        if not os.path.isfile(a_rp.HOSTS_FILE):
+            with open(a_rp.HOSTS_FILE, 'a', encoding='utf-8') as hosts_file:
                 hosts_file.write("# placeholder file for worker DNS entries (see 003-dns)")
 
         ansible_configurator.configure_ansible_yaml(providers=self.providers, configurations=self.configurations,
@@ -324,7 +324,7 @@ def upload_data(self):
     def start_start_instance_threads(self):
         """
         Starts for each provider a start_instances thread and joins them.
-        :return:
+        @return:
         """
         start_instance_threads = []
         for configuration, provider in zip(self.configurations, self.providers):
@@ -338,7 +338,7 @@ def start_start_instance_threads(self):
     def extended_network_configuration(self):
         """
             Configure master/vpn-worker network for a multi/hybrid cloud
-        :return:
+        @return:
         """
         if len(self.providers) == 1:
             return
@@ -368,7 +368,7 @@ def create(self):  # pylint: disable=too-many-branches,too-many-statements
         """
         Creates cluster and logs helpful cluster-info afterwards.
         If debug is set True it offers termination after starting the cluster.
-        :return: exit_state
+        @return: exit_state
         """
         try:
             self.generate_keypair()
@@ -433,7 +433,7 @@ def log_cluster_start_info(self):
         SSH: How to connect to master via SSH
         Terminate: What bibigrid command is needed to terminate the created cluster
         Detailed cluster info: How to log detailed info about the created cluster
-        :return:
+        @return:
         """
         gateway = self.configurations[0].get("gateway")
         ssh_ip = self.master_ip

bibigrid/core/actions/ide.py

@@ -40,7 +40,7 @@ def sigint_handler(caught_signal, frame):  # pylint: disable=unused-argument
 def is_used(ip_address):
     """
     https://stackoverflow.com/questions/62000168/how-to-check-if-ssh-tunnel-is-being-used
-    :return:
+    @return:
     """
     ports_used = []
     with subprocess.Popen(["netstat", "-na"], stdout=subprocess.PIPE) as process:

bibigrid/core/actions/list_clusters.py

@@ -14,9 +14,9 @@
 def dict_clusters(providers, log):
     """
     Creates a dictionary containing all servers by type and provider information
-    :param providers: list of all providers
-    :param log:
-    :return: list of all clusters in yaml format
+    @param providers: list of all providers
+    @param log:
+    @return: list of all clusters in yaml format
     """
     log.info("Creating cluster dictionary...")
     cluster_dict = {}
@@ -39,11 +39,11 @@ def setup(cluster_dict, cluster_id, server, provider):
     """
     Determines cluster_id.
     Generates empty entry for cluster_id in cluster_dict.
-    :param server: found server (dict)
-    :param cluster_id: id of said cluster
-    :param cluster_dict: dict containing all found servers by their cluster_id
-    :param provider: server's provider
-    :return: cluster_id
+    @param server: found server (dict)
+    @param cluster_id: id of said cluster
+    @param cluster_dict: dict containing all found servers by their cluster_id
+    @param provider: server's provider
+    @return: cluster_id
     """
     if not cluster_dict.get(cluster_id):
         cluster_dict[cluster_id] = {}
@@ -57,18 +57,18 @@ def log_list(cluster_id, providers, log):
     """
     Calls dict_clusters and gives a visual representation of the found cluster.
     Detail depends on whether a cluster_id is given or not.
-    :param cluster_id:
-    :param providers:
-    :param log:
-    :return:
+    @param cluster_id:
+    @param providers:
+    @param log:
+    @return:
     """
     cluster_dict = dict_clusters(providers=providers, log=log)
     if cluster_id:  # pylint: disable=too-many-nested-blocks
         if cluster_dict.get(cluster_id):
             log.info("Printing specific cluster_dictionary")
             master_count, worker_count, vpn_count = get_size_overview(cluster_dict[cluster_id], log)
             log.log(42, f"\tCluster has {master_count} master, {vpn_count} vpngtw and {worker_count} regular workers. "
-                       f"The cluster is spread over {vpn_count + master_count} reachable provider(s).")
+                        f"The cluster is spread over {vpn_count + master_count} reachable provider(s).")
             log.log(42, pprint.pformat(cluster_dict[cluster_id]))
         else:
             log.info("Cluster with cluster-id {cluster_id} not found.")
@@ -101,9 +101,9 @@ def log_list(cluster_id, providers, log):
 
 def get_size_overview(cluster_dict, log):
     """
-    :param cluster_dict: dictionary of cluster to size_overview
-    :param log:
-    :return: number of masters, number of workers, number of vpns
+    @param cluster_dict: dictionary of cluster to size_overview
+    @param log:
+    @return: number of masters, number of workers, number of vpns
     """
     log.info("Printing size overview")
     master_count = int(bool(cluster_dict.get("master")))
@@ -115,8 +115,8 @@ def get_size_overview(cluster_dict, log):
 def get_networks(cluster_dict):
     """
     Gets all addresses of servers
-    :param cluster_dict: dictionary of clusters to find addresses
-    :return: dict containing addresses
+    @param cluster_dict: dictionary of clusters to find addresses
+    @return: dict containing addresses
     """
     master = cluster_dict["master"]
     addresses = [{master["provider"]: list(master["addresses"].keys())}]
@@ -128,8 +128,8 @@ def get_networks(cluster_dict):
 def get_security_groups(cluster_dict):
     """
     Gets all security group of servers
-    :param cluster_dict: dictionary of clusters to find security_groups
-    :return: dict containing security_groups
+    @param cluster_dict: dictionary of clusters to find security_groups
+    @return: dict containing security_groups
     """
     master = cluster_dict["master"]
     security_groups = [{master["provider"]: master["security_groups"]}]
@@ -141,10 +141,10 @@ def get_security_groups(cluster_dict):
 def get_master_access_ip(cluster_id, master_provider, log):
     """
     Returns master's ip of cluster cluster_id
-    :param master_provider: master's provider
-    :param cluster_id: Id of cluster
-    :param log:
-    :return: public ip of master
+    @param master_provider: master's provider
+    @param cluster_id: id of cluster
+    @param log:
+    @return: public ip of master
     """
     log.info("Finding master ip for cluster %s...", cluster_id)
     servers = master_provider.list_servers()

bibigrid/core/actions/terminate.py

@@ -57,6 +57,7 @@ def terminate_servers(server_list, cluster_id, provider, log):
     @param server_list: list of server dicts. All servers are from provider
     @param cluster_id: id of cluster to terminate
     @param provider: provider that holds all servers in server_list
+    @param log:
     @return: a list of the servers' (that were to be terminated) termination states
     """
     log.info("Deleting servers on provider %s...", provider.cloud_specification['identifier'])
@@ -73,7 +74,7 @@ def terminate_servers(server_list, cluster_id, provider, log):
 def terminate_server(provider, server, log):
     """
     Terminates a single server and stores the termination state
-    @param provider: the provider that holds the server
+    @param provider: the provider that holds the server.
     @param server: the server that is to be terminated
     @param log:
     @return: true if the server has been terminated, false else

bibigrid/core/actions/update.py

@@ -2,10 +2,10 @@
 Module that contains methods to update the master playbook
 """
 
-from bibigrid.core.utility import ansible_commands as aC
+from bibigrid.core.utility import ansible_commands as a_c
 from bibigrid.core.utility.handler import ssh_handler
-from bibigrid.core.utility.paths import ansible_resources_path as aRP
-from bibigrid.core.utility.paths import bin_path as biRP
+from bibigrid.core.utility.paths import ansible_resources_path as a_rp
+from bibigrid.core.utility.paths import bin_path
 from bibigrid.core.utility.handler import cluster_ssh_handler
 
 
@@ -18,8 +18,8 @@ def update(cluster_id, master_provider, master_configuration, log):
         ssh_handler.execute_ssh(floating_ip=master_ip, private_key=used_private_key, username=ssh_user,
                                 log=log,
                                 gateway=master_configuration.get("gateway", {}),
-                                commands=[aC.EXECUTE],
-                                filepaths=[(aRP.PLAYBOOK_PATH, aRP.PLAYBOOK_PATH_REMOTE),
-                                           (biRP.BIN_PATH, biRP.BIN_PATH_REMOTE)])
+                                commands=[a_c.EXECUTE],
+                                filepaths=[(a_rp.PLAYBOOK_PATH, a_rp.PLAYBOOK_PATH_REMOTE),
+                                           (bin_path.BIN_PATH, bin_path.BIN_PATH_REMOTE)])
         return 0
     return 1