From 35335400d78e78eaf4abe71cdbad073b5f5d9662 Mon Sep 17 00:00:00 2001 From: XaverStiensmeier <36056823+XaverStiensmeier@users.noreply.github.com> Date: Thu, 25 Apr 2024 07:37:06 +0200 Subject: [PATCH] 439 additional ansible roles (#495) * added roles structure * updated roles_path * fixed upper lower case * improved customRole implementation * minor fixes regarding role_paths * improved variable naming of user_roles --- .gitignore | 6 ++++ bibigrid/core/utility/ansible_configurator.py | 32 ++++++++----------- resources/defaults/ansible/ansible.cfg | 1 + .../playbook/roles/additional/tasks/main.yml | 4 --- resources/playbook/roles_galaxy/README | 3 ++ resources/playbook/roles_user/README | 2 ++ .../resistance_nextflow/tasks/main.yml | 25 +++++++++++++++ 7 files changed, 51 insertions(+), 22 deletions(-) delete mode 100644 resources/playbook/roles/additional/tasks/main.yml create mode 100644 resources/playbook/roles_galaxy/README create mode 100644 resources/playbook/roles_user/README create mode 100644 resources/playbook/roles_user/resistance_nextflow/tasks/main.yml diff --git a/.gitignore b/.gitignore index 0f7fbc2f..e2dacc6a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,12 @@ resources/playbook/host_vars/ resources/playbook/group_vars/ resources/tests/bibigrid_test.yml +# Roles +resources/playbook/roles_galaxy/* +!resources/playbook/roles_galaxy/README +resources/playbook/roles_user/* +!resources/playbook/roles_user/README +!resources/playbook/roles_user/resistance_nextflow # any log files *.log log/ diff --git a/bibigrid/core/utility/ansible_configurator.py b/bibigrid/core/utility/ansible_configurator.py index 0ef6a39f..526b80de 100644 --- a/bibigrid/core/utility/ansible_configurator.py +++ b/bibigrid/core/utility/ansible_configurator.py @@ -17,9 +17,8 @@ from bibigrid.core.utility.wireguard import wireguard_keys DEFAULT_NFS_SHARES = ["/vol/spool"] -ADDITIONAL_PATH = "additional/" PYTHON_INTERPRETER = "/usr/bin/python3" -vpngtw_ROLES = [{"role": "bibigrid", "tags": ["bibigrid", "bibigrid-vpngtw"]}] +VPNGTW_ROLES = [{"role": "bibigrid", "tags": ["bibigrid", "bibigrid-vpngtw"]}] MASTER_ROLES = [{"role": "bibigrid", "tags": ["bibigrid", "bibigrid-master"]}] WORKER_ROLES = [{"role": "bibigrid", "tags": ["bibigrid", "bibigrid-worker"]}] VARS_FILES = [aRP.CONFIG_YML, aRP.HOSTS_YML] @@ -48,30 +47,30 @@ def delete_old_vars(log): os.remove(file) -def generate_site_file_yaml(custom_roles): +def generate_site_file_yaml(user_roles): """ Generates site_yaml (dict). Deepcopy is used in case roles might differ between servers in the future. - @param custom_roles: ansibleRoles given by the config + @param user_roles: userRoles given by the config @return: site_yaml (dict) """ site_yaml = [{'hosts': 'master', "become": "yes", "vars_files": VARS_FILES, "roles": MASTER_ROLES}, - {'hosts': 'vpngtw', "become": "yes", "vars_files": VARS_FILES, "roles": vpngtw_ROLES}, - {"hosts": "workers", "become": "yes", "vars_files": VARS_FILES, "roles": WORKER_ROLES}] # , - # {"hosts": "vpngtw", "become": "yes", "vars_files": copy.deepcopy(VARS_FILES), - # "roles": ["common", "vpngtw"]}] - # add custom roles and vars - for custom_role in custom_roles: - VARS_FILES.append(custom_role["vars_file"]) - for role_group in [MASTER_ROLES, vpngtw_ROLES, WORKER_ROLES]: - role_group.append(ADDITIONAL_PATH + custom_role["name"]) + {'hosts': 'vpngtw', "become": "yes", "vars_files": VARS_FILES, "roles": VPNGTW_ROLES}, + {"hosts": "workers", "become": "yes", "vars_files": VARS_FILES, "roles": WORKER_ROLES}] + for user_role in user_roles: + for host_dict in site_yaml: + if host_dict["hosts"] in user_role["hosts"]: + host_dict["vars_files"] = host_dict["vars_files"] + user_role.get("varsFiles", []) + host_dict["roles"] = host_dict["roles"] + [{"role": role["name"], "tags": role.get("tags", [])} for role + in user_role["roles"]] return site_yaml def write_host_and_group_vars(configurations, providers, cluster_id, log): # pylint: disable=too-many-locals """ Filters unnecessary information + @param log: @param configurations: configurations @param providers: providers @param cluster_id: To get proper naming @@ -205,9 +204,6 @@ def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_us master_configuration.get("zabbixConf", {}), strategy=mergedeep.Strategy.TYPESAFE_REPLACE) - for from_key, to_key in [("ansibleRoles", "ansible_roles"), ("ansibleGalaxyRoles", "ansible_galaxy_roles")]: - pass_through(master_configuration, common_configuration_yaml, from_key, to_key) - if len(configurations) > 1: peers = configuration_handler.get_list_by_key(configurations, "wireguard_peer") common_configuration_yaml["wireguard_common"] = {"mask_bits": 24, "listen_port": 51820, "peers": peers} @@ -383,7 +379,7 @@ def configure_ansible_yaml(providers, configurations, cluster_id, log): delete_old_vars(log) log.info("Writing ansible files...") alias = configurations[0].get("aliasDumper", False) - ansible_roles = get_ansible_roles(configurations[0].get("ansibleRoles"), log) + user_roles = configurations[0].get("userRoles") default_user = providers[0].cloud_specification["auth"].get("username", configurations[0].get("sshUser", "Ubuntu")) add_wireguard_peers(configurations) for path, generated_yaml in [ @@ -397,6 +393,6 @@ def configure_ansible_yaml(providers, configurations, cluster_id, log): "sshUser"], configurations, cluster_id, log)), - (aRP.SITE_CONFIG_FILE, generate_site_file_yaml(ansible_roles))]: + (aRP.SITE_CONFIG_FILE, generate_site_file_yaml(user_roles))]: write_yaml(path, generated_yaml, log, alias) write_host_and_group_vars(configurations, providers, cluster_id, log) # writing included in method diff --git a/resources/defaults/ansible/ansible.cfg b/resources/defaults/ansible/ansible.cfg index ee536310..28bf3659 100644 --- a/resources/defaults/ansible/ansible.cfg +++ b/resources/defaults/ansible/ansible.cfg @@ -1,5 +1,6 @@ # This file is moved programmatically to /etc/ansible/ansible.cfg on the master so it shouldn't be moved manually [defaults] +roles_path = "/opt/playbook/roles:/opt/playbook/roles_galaxy:/opt/playbook/roles_user" inventory = ./ansible_hosts host_key_checking = False forks=50 diff --git a/resources/playbook/roles/additional/tasks/main.yml b/resources/playbook/roles/additional/tasks/main.yml deleted file mode 100644 index e949ee7f..00000000 --- a/resources/playbook/roles/additional/tasks/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -- debug: - msg: - - "Hello {{ ansible_user }}!" - diff --git a/resources/playbook/roles_galaxy/README b/resources/playbook/roles_galaxy/README new file mode 100644 index 00000000..efd4f886 --- /dev/null +++ b/resources/playbook/roles_galaxy/README @@ -0,0 +1,3 @@ +# User Roles +This folder can be filled with [galaxy roles](https://docs.ansible.com/ansible/latest/galaxy/user_guide.html). +They will not be overwritten when pulling a new BiBiGrid version. \ No newline at end of file diff --git a/resources/playbook/roles_user/README b/resources/playbook/roles_user/README new file mode 100644 index 00000000..bfb421c8 --- /dev/null +++ b/resources/playbook/roles_user/README @@ -0,0 +1,2 @@ +# User Roles +This folder can be filled with your own custom roles. They will not be overwritten when pulling a new BiBiGrid version. \ No newline at end of file diff --git a/resources/playbook/roles_user/resistance_nextflow/tasks/main.yml b/resources/playbook/roles_user/resistance_nextflow/tasks/main.yml new file mode 100644 index 00000000..85b79975 --- /dev/null +++ b/resources/playbook/roles_user/resistance_nextflow/tasks/main.yml @@ -0,0 +1,25 @@ +- debug: + msg: + - "Hello {{ ansible_user }}!" + +- name: Unarchive ZIP file from GitHub repository + unarchive: + src: "https://github.com/deNBI/bibigrid_clum/raw/main/resources/Resistance_Nextflow.tar.xz" + dest: "/vol/spool/" + remote_src: yes + +- name: Install Java JRE on Debian/Ubuntu + become: True + apt: + name: default-jre # Package name for Java JRE on Debian-based systems + state: present # Ensure that the package is present, you can use "latest" as well + +- name: Get Nextflow + shell: wget -qO- https://get.nextflow.io | bash + args: + chdir: /vol/spool/ + +- name: Execute Nextflow workflow + shell: ./nextflow run resFinder.nf -profile slurm + args: + chdir: "/vol/spool" # Change to the directory where your workflow resides