fix for service being too fast for startup

BiBiServ · May 16, 2024 · 17e89d5 · 17e89d5
1 parent 7c3bb78
commit 17e89d5
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 4 deletions.
diff --git a/bibigrid/core/actions/create.py b/bibigrid/core/actions/create.py
@@ -270,7 +270,13 @@ def initialize_instances(self):
                         "gateway": configuration.get("gateway", {}), "timeout": self.ssh_timeout}
             if configuration.get("masterInstance"):
                 self.master_ip = configuration["floating_ip"]
-                ssh_data["commands"] = self.ssh_add_public_key_commands + ssh_handler.ANSIBLE_SETUP
+                wait_for_service_command, wait_for_service_message = ssh_handler.a_c.WAIT_FOR_SERVICES
+                wait_for_services_commands = [
+                    (wait_for_service_command.format(service=service), wait_for_service_message.format(service=service))
+                    for service in configuration.get("waitForServices", [])]
+                print(wait_for_services_commands)
+                ssh_data["commands"] = (
+                        wait_for_services_commands + self.ssh_add_public_key_commands + ssh_handler.ANSIBLE_SETUP)
                 ssh_data["filepaths"] = [(ssh_data["private_key"], ssh_handler.PRIVATE_KEY_FILE)]
                 ssh_handler.execute_ssh(ssh_data, self.log)
             elif configuration.get("vpnInstance"):
@@ -352,9 +358,9 @@ def upload_data(self):
             self.log.debug(f"Starting playbook with {ansible_start}.")
             commands = [ssh_handler.get_ac_command(self.providers, AC_NAME.format(
                 cluster_id=self.cluster_id))] + ssh_handler.ANSIBLE_START
-        ssh_data = {"floating_ip": self.master_ip, "private_key": KEY_FOLDER + self.key_name,
-                    "username": self.ssh_user, "commands": commands, "filepaths": FILEPATHS,
-                    "gateway": self.configurations[0].get("gateway", {}), "timeout": self.ssh_timeout}
+        ssh_data = {"floating_ip": self.master_ip, "private_key": KEY_FOLDER + self.key_name, "username": self.ssh_user,
+                    "commands": commands, "filepaths": FILEPATHS, "gateway": self.configurations[0].get("gateway", {}),
+                    "timeout": self.ssh_timeout}
         ssh_handler.execute_ssh(ssh_data=ssh_data, log=self.log)
 
     def start_start_server_threads(self):

diff --git a/bibigrid/core/utility/ansible_commands.py b/bibigrid/core/utility/ansible_commands.py
@@ -54,6 +54,9 @@
            "Execute ansible playbook. Be patient.")
 
 # ansible setup
+WAIT_FOR_SERVICES = (
+    "while [[ $(systemctl is-active {service}) == 'active' ]]; do echo 'Waiting for service {service}'; sleep 2; done",
+    "Waiting for service {service}.")
 UPDATE = ("sudo apt-get update", "Update apt repository lists.")
 PYTHON3_PIP = "sudo apt-get install -y python3-pip", "Install python3 pip using apt."
 ANSIBLE_PASSLIB = ("sudo pip install ansible==6.6 passlib", "Install Ansible and Passlib using pip.")