439 additional ansible roles (#495)

* added roles structure

* updated roles_path

* fixed upper lower case

* improved customRole implementation

* minor fixes regarding role_paths

* improved variable naming of user_roles

.gitignore

@@ -12,6 +12,12 @@ resources/playbook/host_vars/
 resources/playbook/group_vars/
 resources/tests/bibigrid_test.yml
 
+# Roles
+resources/playbook/roles_galaxy/*
+!resources/playbook/roles_galaxy/README
+resources/playbook/roles_user/*
+!resources/playbook/roles_user/README
+!resources/playbook/roles_user/resistance_nextflow
 # any log files
 *.log
 log/

bibigrid/core/utility/ansible_configurator.py

@@ -17,9 +17,8 @@
 from bibigrid.core.utility.wireguard import wireguard_keys
 
 DEFAULT_NFS_SHARES = ["/vol/spool"]
-ADDITIONAL_PATH = "additional/"
 PYTHON_INTERPRETER = "/usr/bin/python3"
-vpngtw_ROLES = [{"role": "bibigrid", "tags": ["bibigrid", "bibigrid-vpngtw"]}]
+VPNGTW_ROLES = [{"role": "bibigrid", "tags": ["bibigrid", "bibigrid-vpngtw"]}]
 MASTER_ROLES = [{"role": "bibigrid", "tags": ["bibigrid", "bibigrid-master"]}]
 WORKER_ROLES = [{"role": "bibigrid", "tags": ["bibigrid", "bibigrid-worker"]}]
 VARS_FILES = [aRP.CONFIG_YML, aRP.HOSTS_YML]
@@ -48,30 +47,30 @@ def delete_old_vars(log):
                 os.remove(file)
 
 
-def generate_site_file_yaml(custom_roles):
+def generate_site_file_yaml(user_roles):
     """
     Generates site_yaml (dict).
     Deepcopy is used in case roles might differ between servers in the future.
-    @param custom_roles: ansibleRoles given by the config
+    @param user_roles: userRoles given by the config
     @return: site_yaml (dict)
     """
     site_yaml = [{'hosts': 'master', "become": "yes",
                   "vars_files": VARS_FILES, "roles": MASTER_ROLES},
-                 {'hosts': 'vpngtw', "become": "yes", "vars_files": VARS_FILES, "roles": vpngtw_ROLES},
-                 {"hosts": "workers", "become": "yes", "vars_files": VARS_FILES, "roles": WORKER_ROLES}]  # ,
-    # {"hosts": "vpngtw", "become": "yes", "vars_files": copy.deepcopy(VARS_FILES),
-    # "roles": ["common", "vpngtw"]}]
-    # add custom roles and vars
-    for custom_role in custom_roles:
-        VARS_FILES.append(custom_role["vars_file"])
-        for role_group in [MASTER_ROLES, vpngtw_ROLES, WORKER_ROLES]:
-            role_group.append(ADDITIONAL_PATH + custom_role["name"])
+                 {'hosts': 'vpngtw', "become": "yes", "vars_files": VARS_FILES, "roles": VPNGTW_ROLES},
+                 {"hosts": "workers", "become": "yes", "vars_files": VARS_FILES, "roles": WORKER_ROLES}]
+    for user_role in user_roles:
+        for host_dict in site_yaml:
+            if host_dict["hosts"] in user_role["hosts"]:
+                host_dict["vars_files"] = host_dict["vars_files"] + user_role.get("varsFiles", [])
+                host_dict["roles"] = host_dict["roles"] + [{"role": role["name"], "tags": role.get("tags", [])} for role
+                                                           in user_role["roles"]]
     return site_yaml
 
 
 def write_host_and_group_vars(configurations, providers, cluster_id, log):  # pylint: disable=too-many-locals
     """
     Filters unnecessary information
+    @param log:
     @param configurations: configurations
     @param providers: providers
     @param cluster_id: To get proper naming
@@ -205,9 +204,6 @@ def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_us
                                                                    master_configuration.get("zabbixConf", {}),
                                                                    strategy=mergedeep.Strategy.TYPESAFE_REPLACE)
 
-    for from_key, to_key in [("ansibleRoles", "ansible_roles"), ("ansibleGalaxyRoles", "ansible_galaxy_roles")]:
-        pass_through(master_configuration, common_configuration_yaml, from_key, to_key)
-
     if len(configurations) > 1:
         peers = configuration_handler.get_list_by_key(configurations, "wireguard_peer")
         common_configuration_yaml["wireguard_common"] = {"mask_bits": 24, "listen_port": 51820, "peers": peers}
@@ -383,7 +379,7 @@ def configure_ansible_yaml(providers, configurations, cluster_id, log):
     delete_old_vars(log)
     log.info("Writing ansible files...")
     alias = configurations[0].get("aliasDumper", False)
-    ansible_roles = get_ansible_roles(configurations[0].get("ansibleRoles"), log)
+    user_roles = configurations[0].get("userRoles")
     default_user = providers[0].cloud_specification["auth"].get("username", configurations[0].get("sshUser", "Ubuntu"))
     add_wireguard_peers(configurations)
     for path, generated_yaml in [
@@ -397,6 +393,6 @@ def configure_ansible_yaml(providers, configurations, cluster_id, log):
                                                                                                   "sshUser"],
                                                                                               configurations,
                                                                                               cluster_id, log)),
-        (aRP.SITE_CONFIG_FILE, generate_site_file_yaml(ansible_roles))]:
+        (aRP.SITE_CONFIG_FILE, generate_site_file_yaml(user_roles))]:
         write_yaml(path, generated_yaml, log, alias)
     write_host_and_group_vars(configurations, providers, cluster_id, log)  # writing included in method

resources/defaults/ansible/ansible.cfg

@@ -1,5 +1,6 @@
 # This file is moved programmatically to /etc/ansible/ansible.cfg on the master so it shouldn't be moved manually
 [defaults]
+roles_path = "/opt/playbook/roles:/opt/playbook/roles_galaxy:/opt/playbook/roles_user"
 inventory = ./ansible_hosts
 host_key_checking = False
 forks=50

resources/playbook/roles_galaxy/README

@@ -0,0 +1,3 @@
+# User Roles
+This folder can be filled with [galaxy roles](https://docs.ansible.com/ansible/latest/galaxy/user_guide.html).
+They will not be overwritten when pulling a new BiBiGrid version.

resources/playbook/roles_user/README

@@ -0,0 +1,2 @@
+# User Roles
+This folder can be filled with your own custom roles. They will not be overwritten when pulling a new BiBiGrid version.

resources/playbook/roles_user/resistance_nextflow/tasks/main.yml

@@ -0,0 +1,25 @@
+- debug:
+    msg: 
+    - "Hello {{ ansible_user }}!"
+
+- name: Unarchive ZIP file from GitHub repository
+  unarchive:
+    src: "https://github.com/deNBI/bibigrid_clum/raw/main/resources/Resistance_Nextflow.tar.xz"
+    dest: "/vol/spool/"
+    remote_src: yes
+
+- name: Install Java JRE on Debian/Ubuntu
+  become: True
+  apt:
+    name: default-jre  # Package name for Java JRE on Debian-based systems
+    state: present     # Ensure that the package is present, you can use "latest" as well
+
+- name: Get Nextflow
+  shell: wget -qO- https://get.nextflow.io | bash
+  args:
+    chdir: /vol/spool/
+
+- name: Execute Nextflow workflow
+  shell: ./nextflow run resFinder.nf -profile slurm
+  args:
+    chdir: "/vol/spool"  # Change to the directory where your workflow resides