diff --git a/bibigrid/core/actions/create.py b/bibigrid/core/actions/create.py index b1ca2ee9..c8c8f543 100644 --- a/bibigrid/core/actions/create.py +++ b/bibigrid/core/actions/create.py @@ -141,7 +141,7 @@ def generate_keypair(self): # write cluster_id to automatically read it on following calls if no cid is given with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file: - yaml.safe_dump(data={"cluster_id": self.cluster_id}, stream=cluster_memory_file) + yaml.safe_dump(data={"cluster_id": self.cluster_id, "ssh_user": self.ssh_user}, stream=cluster_memory_file) def delete_old_vars(self): """ @@ -244,7 +244,7 @@ def start_vpn_or_master(self, configuration, provider): # pylint: disable=too-m if identifier == MASTER_IDENTIFIER: with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file: yaml.safe_dump( - data={"cluster_id": self.cluster_id, "floating_ip": configuration["floating_ip"], "name": name}, + data={"cluster_id": self.cluster_id, "floating_ip": configuration["floating_ip"]}, stream=cluster_memory_file) self.log.debug(f"Added floating ip {configuration['floating_ip']} to {name}.") elif identifier == MASTER_IDENTIFIER: diff --git a/requirements-dev.txt b/requirements-dev.txt index 0aee6fe8..2225a96e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ -ansible_lint==24.9.0 -pylint==2.14.5 +ansible-lint==24.10 +pylint==3.3.1 \ No newline at end of file diff --git a/tests/integration/integration_test.py b/tests/integration/integration_test.py new file mode 100644 index 00000000..56175faa --- /dev/null +++ b/tests/integration/integration_test.py @@ -0,0 +1,100 @@ +""" +Integration test module that assumes that one cluster is started at a time. +For multiple clusters the .bibigrid.mem structure needs to be updated TODO +""" + +import os +import subprocess + +import yaml + +# Define common configuration paths +CONFIG_DIR = os.path.expanduser("~/.config/bibigrid") +MEM_FILE = os.path.join(CONFIG_DIR, ".bibigrid.mem") +KEYFILE_PATH_TEMPLATE = os.path.join(CONFIG_DIR, "keys", "tempKey_bibi-{cluster_id}") +BIBIGRID_SCRIPT = os.path.abspath("../../bibigrid.sh") + + +def start_cluster(): + """Start the cluster by calling bibigrid.sh.""" + print("Starting the cluster...") + result = subprocess.run([BIBIGRID_SCRIPT, "-c", "-vv", "-i", "bibigrid.yaml"], capture_output=True, text=True, + check=False) + if result.returncode == 0: + print("Cluster started successfully.") + # print(result.stdout) + else: + print("Failed to start the cluster.") + # print(result.stderr) + raise Exception("Cluster start failed") + + +def read_cluster_info(): + """Read last cluster information from bibigrid.mem file.""" + with open(MEM_FILE, "r", encoding="utf8") as f: + cluster_data = yaml.safe_load(f) + return cluster_data["cluster_id"], cluster_data["floating_ip"], cluster_data["ssh_user"] + + +def build_keyfile_path(cluster_id): + """Construct the keyfile path using cluster ID.""" + return KEYFILE_PATH_TEMPLATE.format(cluster_id=cluster_id) + + +def ssh_command(master_ip, keyfile, command, ssh_user): + """Execute a command on the master node via SSH.""" + ssh_cmd = [ + "ssh", + "-i", keyfile, + "-o", "StrictHostKeyChecking=no", + f"{ssh_user}@{master_ip}", + command + ] + return subprocess.run(ssh_cmd, capture_output=True, text=True, check=False) + + +def terminate_cluster(): + """Terminate the cluster by calling bibigrid.sh.""" + print("Terminating the cluster...") + result = subprocess.run([BIBIGRID_SCRIPT, "-i", "bibigrid.yaml", "-t", "-vv"], capture_output=True, text=True, + check=False) + if result.returncode == 0: + print("Cluster terminated successfully.") + print(result.stdout) + else: + print("Failed to terminate the cluster.") + print(result.stderr) + raise Exception("Cluster termination failed") + + +def main(): + # Step 0: Create the cluster + start_cluster() + # Step 1: Read cluster info + cluster_id, master_ip, ssh_user = read_cluster_info() + print(f"Cluster ID: {cluster_id}, Master IP: {master_ip}") + + # Step 2: Build keyfile path + keyfile = build_keyfile_path(cluster_id) + print(f"Using keyfile: {keyfile}") + + # Step 3: Check worker nodes by running srun from master node + check_command = "srun -N2 hostname>" + print(f"Running on master: {check_command}") + + # Run the command on the master instance + result = ssh_command(master_ip, keyfile, check_command, ssh_user) + + if result.returncode == 0: + print("Worker nodes are up and responding:") + print(result.stdout) + else: + print("Failed to run command on worker nodes.") + print(result.stderr) + + # Step N: Terminate the cluster + terminate_cluster() + + +if __name__ == "__main__": + main() diff --git a/tests/provider/test_provider.py b/tests/unittests/provider/test_provider.py similarity index 100% rename from tests/provider/test_provider.py rename to tests/unittests/provider/test_provider.py diff --git a/tests/startup_tests.py b/tests/unittests/startup_tests.py similarity index 88% rename from tests/startup_tests.py rename to tests/unittests/startup_tests.py index 4a719ef2..494b9533 100644 --- a/tests/startup_tests.py +++ b/tests/unittests/startup_tests.py @@ -34,12 +34,12 @@ def suppress_stdout(): logging.basicConfig(level=logging.ERROR) if __name__ == '__main__': # Unittests - suite = unittest.TestLoader().discover("./", pattern='test_*.py') + suite = unittest.TestLoader().discover("../", pattern='test_*.py') with suppress_stdout(): unittest.TextTestRunner(verbosity=2).run(suite) # Provider-Test # Configuration needs to contain providers and infrastructures - suite = unittest.TestLoader().discover("./provider", pattern='test_*.py') + suite = unittest.TestLoader().discover("provider", pattern='test_*.py') with suppress_stdout(): unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_check.py b/tests/unittests/test_check.py similarity index 100% rename from tests/test_check.py rename to tests/unittests/test_check.py diff --git a/tests/test_configuration_handler.py b/tests/unittests/test_configuration_handler.py similarity index 100% rename from tests/test_configuration_handler.py rename to tests/unittests/test_configuration_handler.py diff --git a/tests/test_id_generation.py b/tests/unittests/test_id_generation.py similarity index 100% rename from tests/test_id_generation.py rename to tests/unittests/test_id_generation.py diff --git a/tests/test_list_clusters.py b/tests/unittests/test_list_clusters.py similarity index 100% rename from tests/test_list_clusters.py rename to tests/unittests/test_list_clusters.py diff --git a/tests/test_provider_handler.py b/tests/unittests/test_provider_handler.py similarity index 100% rename from tests/test_provider_handler.py rename to tests/unittests/test_provider_handler.py diff --git a/tests/test_return_threading.py b/tests/unittests/test_return_threading.py similarity index 100% rename from tests/test_return_threading.py rename to tests/unittests/test_return_threading.py diff --git a/tests/test_ssh_handler.py b/tests/unittests/test_ssh_handler.py similarity index 100% rename from tests/test_ssh_handler.py rename to tests/unittests/test_ssh_handler.py diff --git a/tests/test_startup.py b/tests/unittests/test_startup.py similarity index 100% rename from tests/test_startup.py rename to tests/unittests/test_startup.py diff --git a/tests/test_terminate_cluster.py b/tests/unittests/test_terminate_cluster.py similarity index 100% rename from tests/test_terminate_cluster.py rename to tests/unittests/test_terminate_cluster.py diff --git a/tests/test_validate_configuration.py b/tests/unittests/test_validate_configuration.py similarity index 100% rename from tests/test_validate_configuration.py rename to tests/unittests/test_validate_configuration.py diff --git a/tests/test_validate_schema.py b/tests/unittests/test_validate_schema.py similarity index 100% rename from tests/test_validate_schema.py rename to tests/unittests/test_validate_schema.py