From 1085de4173b230fb4763895a6d0a4379f02f220b Mon Sep 17 00:00:00 2001 From: Mark Wong Date: Tue, 17 Oct 2023 22:12:22 +0000 Subject: [PATCH] Update dbt7 roles to use AppImage --- roles/setup_dbt7/README.md | 48 +++++-------------- roles/setup_dbt7/defaults/main.yml | 14 ++++-- .../tasks/dbt7_configure_profile.yml | 7 --- .../tasks/dbt7_install_packages.yml | 27 ++++++----- roles/setup_dbt7/tasks/dbt7_tpc.yml | 25 +++------- roles/setup_dbt7/tasks/main.yml | 8 +--- .../setup_dbt7/tasks/validate_setup_dbt7.yml | 48 ++++++++----------- roles/setup_dbt7/templates/dbt7_profile | 19 -------- roles/setup_dbt7/vars/DBaaS.yml | 4 -- roles/setup_dbt7/vars/EPAS.yml | 5 +- roles/setup_dbt7/vars/PG.yml | 5 +- tests/cases/setup_dbt7/vars.json | 3 +- tests/tests/test_setup_dbt7.py | 18 ++++++- 13 files changed, 86 insertions(+), 145 deletions(-) delete mode 100644 roles/setup_dbt7/tasks/dbt7_configure_profile.yml delete mode 100644 roles/setup_dbt7/templates/dbt7_profile delete mode 100644 roles/setup_dbt7/vars/DBaaS.yml diff --git a/roles/setup_dbt7/README.md b/roles/setup_dbt7/README.md index c25b34c6c..4e605d1e6 100644 --- a/roles/setup_dbt7/README.md +++ b/roles/setup_dbt7/README.md @@ -6,47 +6,31 @@ This role is for installing DBT-7. Following are the requirements of this role. 1. Ansible - 2. `edb_devops.edb_postgres` -> `setup_repo` role for setting the repository on - the systems. ## Role Variables When executing the role via ansible these are the required variables: - * ***dbt7_pgdata*** + * ***dbt7_version*** - This variable is to be used to specify a custom PGDATA directory that DBT-7 - will use separately from the default PGDATA that may be used in other - playbooks. Thus do not set this to a directory that may be used elsewhere. - You are not likely to use this unless you are testing specific source based - installations of PostgreSQL or doing any other highly customized testing with - logical or physical system configurations. + These playbooks can install any version of the DBT-7 AppImage that is + packaged from GitHub. See the following link for available versions: + https://github.com/osdldbt/dbt7/releases - * ***dbt7-version*** + * ***have_tpcdsfile*** - These playbooks can install any version of DBT-7 that is packaged from GitHub. - See the following link for available versions: - https://github.com/osdldbt/dbt7-packaging/releases - - * ***have_tpcfile*** - - The is a boolean use for CI purposes where we cannot redistribute TPC code + This is a boolean used for CI purposes where we cannot redistribute TPC code thus the CI system will not attempt to run the tasks specific for setting up the TPC code. - * ***pg_version*** - - Postgres Versions supported are: 10, 11, 12, 13 and 14 - - * ***tpcfile*** + * ***tpcdsfile*** This is the full path to the TPC-DS Tools zip file that must be downloaded by - the you, the user, per TPC EULA agreement. The TPC-DS Tools can be requested - at: + you, the user, per TPC EULA agreement. The TPC-DS Tools can be requested at: https://www.tpc.org/tpc_documents_current_versions/current_specifications5.asp These variables can be assigned in the `pre_tasks` definition of the -section: *How to include the `setup_dbt7` role in your Playbook*. +section. ## Example Playbook @@ -74,7 +58,7 @@ Below is an example of how to include the `setup_dbt7` role: ```yaml --- - hosts: all - name: Postgres deployment playbook for DBT-3 + name: Postgres deployment playbook for DBT-7 become: yes gather_facts: yes any_errors_fatal: True @@ -84,27 +68,19 @@ Below is an example of how to include the `setup_dbt7` role: - edb_devops.edb_postgres roles: - - role: setup_repo - when: "'setup_repo' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" - - role: install_dbserver - when: "'install_dbserver' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" - - role: init_dbserver - when: "'init_dbserver' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" - role: setup_dbt7 when: "'setup_dbt7' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" - - role: tuning - when: "'tuning' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" ``` ## Playbook execution examples ```bash -# To deploy community Postgres version 13 with the user centos +# To deploy community Postgres with the user centos $ ansible-playbook playbook.yml \ -i inventory.yml \ -u centos \ --private-key \ - --extra-vars="pg_version=13 pg_type=PG" + --extra-vars="pg_type=PG" ``` ## License diff --git a/roles/setup_dbt7/defaults/main.yml b/roles/setup_dbt7/defaults/main.yml index 672b33939..93a48a50c 100644 --- a/roles/setup_dbt7/defaults/main.yml +++ b/roles/setup_dbt7/defaults/main.yml @@ -1,9 +1,12 @@ --- -dbt7_version: "0.4.1" -dbt7_pgdata: "/pgdata/dbt7/pgdata" -pg_version: 14 -have_tpcfile: true -tpcfile: "" +dbt7_version: "0.5.0" +have_tpcdsfile: true +tpcdsfile: "" + +# DBT-7 AppImage information +dbt7_appimage: "https://github.com/osdldbt/dbt7/releases/download/v{{ dbt7_version }}/dbt7-{{ dbt7_version }}-x86_64.AppImage" +dbt7_checksum: "{{ dbt7_appimage }}.sha256" +dbt7_path: "/usr/bin/dbt7" # setting validate_only to true allows you to validate setup on an existing node # use_validation flag applies to deployment configuration and validation after setup @@ -15,3 +18,4 @@ supported_os: - RedHat8 - Rocky8 - AlmaLinux8 + - OracleLinux7 diff --git a/roles/setup_dbt7/tasks/dbt7_configure_profile.yml b/roles/setup_dbt7/tasks/dbt7_configure_profile.yml deleted file mode 100644 index 1d365f09b..000000000 --- a/roles/setup_dbt7/tasks/dbt7_configure_profile.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Install DBT-7 profile - ansible.builtin.template: - dest: "{{ pg_user_home }}/.dbt7_profile" - mode: "0644" - src: dbt7_profile - become_user: "{{ pg_owner }}" diff --git a/roles/setup_dbt7/tasks/dbt7_install_packages.yml b/roles/setup_dbt7/tasks/dbt7_install_packages.yml index dc433da17..1086c7dcf 100644 --- a/roles/setup_dbt7/tasks/dbt7_install_packages.yml +++ b/roles/setup_dbt7/tasks/dbt7_install_packages.yml @@ -1,27 +1,30 @@ --- - - name: Install additional supporting packages ansible.builtin.package: name: - - bison - - gcc + - bc + - byacc + - dejavu-fonts-common - flex + - fontconfig + - fuse + - fuse-libs + - gcc + - make - patch - perf - psmisc - - python3-docutils - - R-core - sysstat - tmux + - unzip state: present when: ansible_os_family == "RedHat" become: true -- name: Install packages for DBT-7 Database - ansible.builtin.dnf: - name: - - 'https://github.com/osdldbt/dbt7-packaging/releases/download/v{{ dbt7_version }}/dbt7-{{ dbt7_version }}-1.el8.x86_64.rpm' - state: present - disable_gpg_check: true - when: ansible_os_family == "RedHat" +- name: Download DBT-7 AppImage + ansible.builtin.get_url: + url: "{{ dbt7_appimage }}" + checksum: "sha256:{{ dbt7_checksum }}" + dest: "{{ dbt7_path }}" + mode: '0755' become: true diff --git a/roles/setup_dbt7/tasks/dbt7_tpc.yml b/roles/setup_dbt7/tasks/dbt7_tpc.yml index fb7efb325..6bf77461a 100644 --- a/roles/setup_dbt7/tasks/dbt7_tpc.yml +++ b/roles/setup_dbt7/tasks/dbt7_tpc.yml @@ -1,30 +1,19 @@ --- -- name: Check that 'tpcfile' is set to local TPC-H Tools location +- name: Check that 'tpcdsfile' is set to local TPC-H Tools location ansible.builtin.fail: msg: >- - The Ansible variable 'tpcfile' must be set to a locally downloaded copy + The Ansible variable 'tpcdsfile' must be set to a locally downloaded copy of the TPC supplied TPC-H Tools from: https://www.tpc.org/tpc_documents_current_versions/current_specifications5.asp - when: tpcfile|length < 1 + when: tpcdsfile | length < 1 - name: Copy and unarchive TPC-H Tools to remote host ansible.builtin.unarchive: - src: "{{ tpcfile }}" + src: "{{ tpcdsfile }}" dest: "{{ pg_user_home }}" become_user: "{{ pg_owner }}" -- name: Patch TPC-H Tools for PostgreSQL - ansible.posix.patch: - remote_src: true - src: "{{ item }}" - basedir: "{{ pg_user_home }}" - strip: 1 - become_user: "{{ pg_owner }}" - loop: - - /usr/share/dbt7/dbt7-DSGen-software-code-3.2.0rc1.diff - - /usr/share/dbt7/dbt7-DSGen-software-code-3.2.0rc1-postgresql-queries.diff - -- name: Build TPC-H Tools for PostgreSQL - community.general.make: - chdir: "{{ pg_user_home }}/DSGen-software-code-3.2.0rc1/tools" +- name: Build TPC-DS Tools for PostgreSQL + ansible.builtin.shell: >- + dbt7 build-dsgen {{ pg_user_home }}/DSGen-software-code-3.2.0rc1 become_user: "{{ pg_owner }}" diff --git a/roles/setup_dbt7/tasks/main.yml b/roles/setup_dbt7/tasks/main.yml index 0e7cabec8..ed16d9e6e 100644 --- a/roles/setup_dbt7/tasks/main.yml +++ b/roles/setup_dbt7/tasks/main.yml @@ -8,12 +8,8 @@ - name: Include TPC installation ansible.builtin.include_tasks: dbt7_tpc.yml - when: have_tpcfile and not validate_only|bool + when: have_tpcdsfile and not validate_only|bool -- name: Include DBT-7 profile configuration - ansible.builtin.include_tasks: dbt7_configure_profile.yml - when: not validate_only|bool - -- name: Include validate_setup_dbt7 tasks +- name: Include the validate_setup_dbt7 tasks ansible.builtin.include_tasks: validate_setup_dbt7.yml when: validate_only|bool or use_validation|bool diff --git a/roles/setup_dbt7/tasks/validate_setup_dbt7.yml b/roles/setup_dbt7/tasks/validate_setup_dbt7.yml index 835eb2805..4e308074b 100644 --- a/roles/setup_dbt7/tasks/validate_setup_dbt7.yml +++ b/roles/setup_dbt7/tasks/validate_setup_dbt7.yml @@ -1,45 +1,39 @@ --- -# validate dbt7 package -- name: Set dbt7 package name +# validate dbt7 db packages installation +- name: Set dbt7_db_pkg_list ansible.builtin.set_fact: - dbt7_package: 'dbt7' + dbt7_db_pkg_list: ['bc', 'byacc', 'dejavu-fonts-common', 'flex', 'fontconfig', 'fuse', 'fuse-libs', 'gcc', 'make', 'patch', 'perf', 'psmisc', 'sysstat', 'tmux', 'unzip'] - name: Gather the package facts ansible.builtin.package_facts: strategy: "all" -- name: Check that DBT7 packages have been installed +- name: Check that all packages in dbt7_db_pkg_list has been installed ansible.builtin.assert: that: - - "{{ ansible_facts.packages[dbt7_package] }} is defined" - fail_msg: "Package {{ dbt7_package }} has not been installed." - success_msg: "Package {{ dbt7_package }} has been installed." + - "{{ ansible_facts.packages[package] }} is defined" + fail_msg: "Package {{ package }} has not been installed." + success_msg: "Package {{ package }} has been installed." + loop: "{{ dbt7_db_pkg_list }}" + loop_control: + loop_var: package -# validate dbt7 user -- name: Run query to check dbt7 user is in pg_user - ansible.builtin.include_role: - name: manage_dbserver - tasks_from: execute_sql_scripts - vars: - pg_query: - - query: "Select * from pg_user where usename = '{{ pg_owner }}'" - db: "{{ pg_database }}" - -- name: Set user_query_result with sql_query_output - ansible.builtin.set_fact: - user_query_result: "{{ sql_query_output }}" +# ensure the DBT-7 AppImage has been installed to the proper location +- name: Stat to dbt7_path + ansible.builtin.stat: + path: "{{ dbt7_path }}" + register: dbt7_path_stat become: true -- name: Check if DBT7 postgres user was successfully created +- name: Check if DBT-7 AppImage was installed properly on DB ansible.builtin.assert: that: - - user_query_result.results[0].query_result[0]['usename'] == pg_owner - fail_msg: "DBT7 postgres user was not successfully created." - success_msg: "DBT7 postgres user was successfully created." - run_once: true + - dbt7_path_stat.stat['exists']|bool + fail_msg: "DBT-7 AppImage at {{ dbt7_path }} was not created properly" + success_msg: "DBT-7 AppImage at {{ dbt7_path }} was created successfully" # reset variables - name: Reset variables ansible.builtin.set_fact: - dbt7_package: null - user_query_result: null + dbt7_db_pkg_list: null + dbt7_path_stat: null diff --git a/roles/setup_dbt7/templates/dbt7_profile b/roles/setup_dbt7/templates/dbt7_profile deleted file mode 100644 index 78ac99f8a..000000000 --- a/roles/setup_dbt7/templates/dbt7_profile +++ /dev/null @@ -1,19 +0,0 @@ -export DSHOME="{{ pg_user_home}}/DSGen-software-code-3.2.0rc1" -export DSQUERY="${DSHOME}/query_templates" -export DSTOOLS="${DSHOME}/tools" -export DS_DATA="/tmp/dbt7data" - -export DBT7PGDATA="{{ dbt7_pgdata }}" -export DBT7DBNAME=dbt7 - -# Set PGHOST to connect to a specific hostname or Unix-domain socket. -# Set PGPORT to connect to a non-default TCP port. - -export DEFAULT_LOAD_PARAMETERS="" -export DEFAULT_PERF_PARAMETERS="" - -# Uncomment one of the following to generate the desired EXPLAIN output with the -# data maintenance test. Otherwise the DELETE and INSERT statements will -# execute without any EXPLAIN output. -#export DBT7DMPREFIX="LOAD 'auto_explain'; SET auto_explain.log_min_duration TO 0; SET auto_explain.log_level TO notice;" -#export DBT7DMPREFIX="LOAD 'auto_explain'; SET auto_explain.log_min_duration TO 0; SET auto_explain.log_analyze TO on; SET auto_explain.log_level TO notice;" diff --git a/roles/setup_dbt7/vars/DBaaS.yml b/roles/setup_dbt7/vars/DBaaS.yml deleted file mode 100644 index 462d34eba..000000000 --- a/roles/setup_dbt7/vars/DBaaS.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -pg_port: "5432" -pg_superuser: "postgres" -pg_superuser_password: "postgres" diff --git a/roles/setup_dbt7/vars/EPAS.yml b/roles/setup_dbt7/vars/EPAS.yml index 6618fed0e..93cfd4cc0 100644 --- a/roles/setup_dbt7/vars/EPAS.yml +++ b/roles/setup_dbt7/vars/EPAS.yml @@ -1,6 +1,3 @@ --- pg_owner: "enterprisedb" -pg_port: "5444" -pg_superuser: "enterprisedb" -pg_instance_name: "main" -pg_data: "/var/lib/edb/as{{ pg_version }}/{{ pg_instance_name }}/data" +pg_user_home: "/var/lib/edb" diff --git a/roles/setup_dbt7/vars/PG.yml b/roles/setup_dbt7/vars/PG.yml index 15ed33273..642db9a9e 100644 --- a/roles/setup_dbt7/vars/PG.yml +++ b/roles/setup_dbt7/vars/PG.yml @@ -1,6 +1,3 @@ --- pg_owner: "postgres" -pg_port: "5432" -pg_superuser: "postgres" -pg_instance_name: "main" -pg_data: "/var/lib/pgsql/{{ pg_version }}/{{ pg_instance_name }}/data" +pg_user_home: "/var/lib/pgsql" diff --git a/tests/cases/setup_dbt7/vars.json b/tests/cases/setup_dbt7/vars.json index 0ac512cd3..d528d5ffb 100644 --- a/tests/cases/setup_dbt7/vars.json +++ b/tests/cases/setup_dbt7/vars.json @@ -2,6 +2,5 @@ "use_hostname": false, "pg_data": "/opt/pgdata", "pg_wal": "/opt/pgwal", - "limit_file_path": "/etc/security/limits.conf", - "have_tpcfile": false + "have_tpcdsfile": false } diff --git a/tests/tests/test_setup_dbt7.py b/tests/tests/test_setup_dbt7.py index 89786f2b2..27a52870b 100644 --- a/tests/tests/test_setup_dbt7.py +++ b/tests/tests/test_setup_dbt7.py @@ -10,7 +10,23 @@ def test_setup_dbt7_packages(): host = get_primary() - packages = ['dbt7'] + packages = [ + 'bc', + 'byacc', + 'dejavu-fonts-common', + 'flex', + 'fontconfig', + 'fuse', + 'fuse-libs', + 'gcc', + 'make', + 'patch', + 'perf', + 'psmisc', + 'sysstat', + 'tmux', + 'unzip', + ] for package in packages: assert host.package(package).is_installed, \