diff --git a/roles/setup_dbt3/README.md b/roles/setup_dbt3/README.md index e8dde9c04..bf6715dad 100644 --- a/roles/setup_dbt3/README.md +++ b/roles/setup_dbt3/README.md @@ -2,53 +2,35 @@ This role is for installing DBT-3. - - ## Requirements Following are the requirements of this role. 1. Ansible - 2. `edb_devops.edb_postgres` -> `setup_repo` role for setting the repository on - the systems. ## Role Variables When executing the role via ansible these are the required variables: - * ***dbt3_pgdata*** - - This variable is to be used to specify a custom PGDATA directory that DBT-3 - will use separately from the default PGDATA that may be used in other - playbooks. Thus do not set this to a directory that may be used elsewhere. - You are not likely to use this unless you are testing specific source based - installations of PostgreSQL or doing any other highly customized testing with - logical or physical system configurations. - * ***dbt3_version*** - These playbooks can install any version of DBT-3 that is packaged from GitHub. - See the following link for available versions: - https://github.com/osdldbt/dbt3-packaging/releases + These playbooks can install any version of the DBT-3 AppImage that is + packaged from GitHub. See the following link for available versions: + https://github.com/osdldbt/dbt3/releases * ***have_tpcfile*** - The is a boolean use for CI purposes where we cannot redistribute TPC code + This is a boolean used for CI purposes where we cannot redistribute TPC code thus the CI system will not attempt to run the tasks specific for setting up the TPC code. - * ***pg_version*** - - Postgres Versions supported are: 10, 11, 12, 13 and 14 - * ***tpcfile*** This is the full path to the TPC-H Tools zip file that must be downloaded by - the you, the user, per TPC EULA agreement. The TPC-H Tools can be requested - at: + you, the user, per TPC EULA agreement. The TPC-H Tools can be requested at: https://www.tpc.org/tpc_documents_current_versions/current_specifications5.asp These variables can be assigned in the `pre_tasks` definition of the -section: *How to include the `setup_dbt3` role in your Playbook*. +section. ## Example Playbook @@ -86,16 +68,8 @@ Below is an example of how to include the `setup_dbt3` role: - edb_devops.edb_postgres roles: - - role: setup_repo - when: "'setup_repo' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" - - role: install_dbserver - when: "'install_dbserver' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" - - role: init_dbserver - when: "'init_dbserver' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" - role: setup_dbt3 when: "'setup_dbt3' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" - - role: tuning - when: "'tuning' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)" ``` ## Playbook execution examples @@ -106,7 +80,7 @@ $ ansible-playbook playbook.yml \ -i inventory.yml \ -u centos \ --private-key \ - --extra-vars="pg_version=13 pg_type=PG" + --extra-vars="pg_type=PG" ``` ## License diff --git a/roles/setup_dbt3/defaults/main.yml b/roles/setup_dbt3/defaults/main.yml index 1838f69db..3f893a759 100644 --- a/roles/setup_dbt3/defaults/main.yml +++ b/roles/setup_dbt3/defaults/main.yml @@ -1,10 +1,13 @@ --- -dbt3_version: "1.11.0" -dbt3_pgdata: "/pgdata/dbt3/pgdata" -pg_version: 14 +dbt3_version: "2.0.1" have_tpcfile: true tpcfile: "" +# DBT-3 AppImage information +dbt3_appimage: "https://github.com/osdldbt/dbt3/releases/download/v{{ dbt3_version }}/dbt3-{{ dbt3_version }}-x86_64.AppImage" +dbt3_checksum: "{{ dbt3_appimage }}.sha256" +dbt3_path: "/usr/bin/dbt3" + # setting validate_only to true allows you to validate setup on an existing node # use_validation flag applies to deployment configuration and validation after setup validate_only: false diff --git a/roles/setup_dbt3/tasks/dbt3_configure_profile.yml b/roles/setup_dbt3/tasks/dbt3_configure_profile.yml deleted file mode 100644 index 03fb1e9b6..000000000 --- a/roles/setup_dbt3/tasks/dbt3_configure_profile.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Install DBT-3 profile - ansible.builtin.template: - dest: "{{ pg_user_home }}/.dbt3_profile" - mode: "0644" - src: dbt3_profile - become_user: "{{ pg_owner }}" diff --git a/roles/setup_dbt3/tasks/dbt3_install_packages.yml b/roles/setup_dbt3/tasks/dbt3_install_packages.yml index 00a374dae..1e72741e3 100644 --- a/roles/setup_dbt3/tasks/dbt3_install_packages.yml +++ b/roles/setup_dbt3/tasks/dbt3_install_packages.yml @@ -3,23 +3,26 @@ ansible.builtin.package: name: - bc + - dejavu-fonts-common + - fontconfig + - fuse + - fuse-libs - gcc + - make - patch - perf - psmisc - - python3-docutils - - R-core - sysstat - tmux + - unzip state: present when: ansible_os_family == "RedHat" become: true -- name: Install packages for DBT-3 Database - ansible.builtin.dnf: - name: - - 'https://github.com/osdldbt/dbt3-packaging/releases/download/v{{ dbt3_version }}/dbt3-{{ dbt3_version }}-1.el8.x86_64.rpm' - state: present - disable_gpg_check: true - when: ansible_os_family == "RedHat" +- name: Download DBT-3 AppImage + ansible.builtin.get_url: + url: "{{ dbt3_appimage }}" + checksum: "sha256:{{ dbt3_checksum }}" + dest: "{{ dbt3_path }}" + mode: '0755' become: true diff --git a/roles/setup_dbt3/tasks/dbt3_tpc.yml b/roles/setup_dbt3/tasks/dbt3_tpc.yml index cc1aa8a6b..ea04063f5 100644 --- a/roles/setup_dbt3/tasks/dbt3_tpc.yml +++ b/roles/setup_dbt3/tasks/dbt3_tpc.yml @@ -13,19 +13,7 @@ dest: "{{ pg_user_home }}" become_user: "{{ pg_owner }}" -- name: Patch TPC-H Tools for PostgreSQL - ansible.posix.patch: - remote_src: true - src: "{{ item }}" - basedir: "{{ pg_user_home }}" - strip: 1 - become_user: "{{ pg_owner }}" - loop: - - /usr/share/dbt3/dbt3-TPC-H_Tools_v3.0.0-postgresql.diff - - /usr/share/dbt3/dbt3-TPC-H_Tools_v3.0.0-stdout.diff - - name: Build TPC-H Tools for PostgreSQL - community.general.make: - chdir: "{{ pg_user_home }}/TPC-H_Tools_v3.0.0/dbgen" - file: Makefile.postgresql + ansible.builtin.shell: >- + dbt3 build-dbgen pgsql {{ pg_user_home }}/'TPC-H V3.0.1' become_user: "{{ pg_owner }}" diff --git a/roles/setup_dbt3/tasks/main.yml b/roles/setup_dbt3/tasks/main.yml index 3bd2421e7..4776914de 100644 --- a/roles/setup_dbt3/tasks/main.yml +++ b/roles/setup_dbt3/tasks/main.yml @@ -10,10 +10,6 @@ ansible.builtin.include_tasks: dbt3_tpc.yml when: have_tpcfile and not validate_only|bool -- name: Include DBT-3 profile configuration - ansible.builtin.include_tasks: dbt3_configure_profile.yml - when: not validate_only|bool - - name: Include the validate_setup_dbt3 tasks ansible.builtin.include_tasks: validate_setup_dbt3.yml when: validate_only|bool or use_validation|bool diff --git a/roles/setup_dbt3/tasks/validate_setup_dbt3.yml b/roles/setup_dbt3/tasks/validate_setup_dbt3.yml index 3f4726168..a4bd18f17 100644 --- a/roles/setup_dbt3/tasks/validate_setup_dbt3.yml +++ b/roles/setup_dbt3/tasks/validate_setup_dbt3.yml @@ -1,45 +1,40 @@ --- -# validate dbt3 packages -- name: Set dbt3 package name +# validate dbt3 db packages installation +- name: Set dbt3_db_pkg_list ansible.builtin.set_fact: - dbt3_package: 'dbt3' + dbt3_db_pkg_list: [ 'bc', 'dejavu-fonts-common', 'fontconfig', 'fuse', 'fuse-libs', 'gcc', 'make', 'patch', 'perf', 'psmisc', 'sysstat', 'tmux', 'unzip' ] - name: Gather the package facts ansible.builtin.package_facts: strategy: "all" -- name: Check that DBT3 packages have been installed +- name: Check that all packages in dbt3_db_pkg_list has been installed ansible.builtin.assert: that: - - "{{ ansible_facts.packages[dbt3_package] }} is defined" - fail_msg: "Package {{ dbt3_package }} has not been installed." - success_msg: "Package {{ dbt3_package }} has been installed." + - "{{ ansible_facts.packages[package] }} is defined" + fail_msg: "Package {{ package }} has not been installed." + success_msg: "Package {{ package }} has been installed." + loop: "{{ dbt3_db_pkg_list }}" + loop_control: + loop_var: package -# validate dbt3 user -- name: Run query to check dbt3 user is in pg_user - ansible.builtin.include_role: - name: manage_dbserver - tasks_from: execute_sql_scripts - vars: - pg_query: - - query: "Select * from pg_user where usename = '{{ pg_owner }}'" - db: "{{ pg_database }}" - -- name: Set user_query_result with sql_query_output - ansible.builtin.set_fact: - user_query_result: "{{ sql_query_output }}" +# ensure the DBT-3 AppImage has been installed to the proper location +- name: Stat to dbt3_path + ansible.builtin.stat: + path: "{{ dbt3_path }}" + register: dbt3_path_stat become: true -- name: Check if DBT3 postgres user was successfully created +- name: Check if DBT-3 AppImage was installed properly on DB ansible.builtin.assert: that: - - user_query_result.results[0].query_result[0]['usename'] == pg_owner - fail_msg: "DBT3 postgres user was not successfully created." - success_msg: "DBT3 postgres user was successfully created." - run_once: true + - dbt3_path_stat.stat['exists']|bool + fail_msg: "DBT-3 AppImage at {{ dbt3_path }} was not created properly" + success_msg: "DBT-3 AppImage at {{ dbt3_path }} was created successfully" # reset variables - name: Reset variables ansible.builtin.set_fact: - dbt3_package: null + dbt3_db_pkg_list: null + dbt3_path_stat: null user_query_result: null diff --git a/roles/setup_dbt3/templates/dbt3_profile b/roles/setup_dbt3/templates/dbt3_profile deleted file mode 100644 index 0916ea920..000000000 --- a/roles/setup_dbt3/templates/dbt3_profile +++ /dev/null @@ -1,15 +0,0 @@ -export DSS_PATH="/tmp/dss" -export DSS_QUERY="/usr/local/share/dbt3/queries/pgsql" -export DSS_CONFIG="{{ pg_user_home}}/TPC-H_Tools_v3.0.0/dbgen" - -export DBGEN="$DSS_CONFIG/dbgen" -export QGEN="$DSS_CONFIG/qgen" - -export PGDATABASE=dbt3 -export PGDATA={{ dbt3_pgdata }} - -export PGPORT=5432 - -export DEFAULT_LOAD_PARAMETERS="" -export DEFAULT_POWER_PARAMETERS="" -export DEFAULT_THROUGHPUT_PARAMETERS="" diff --git a/roles/setup_dbt3/vars/DBaaS.yml b/roles/setup_dbt3/vars/DBaaS.yml deleted file mode 100644 index 462d34eba..000000000 --- a/roles/setup_dbt3/vars/DBaaS.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -pg_port: "5432" -pg_superuser: "postgres" -pg_superuser_password: "postgres" diff --git a/roles/setup_dbt3/vars/EPAS.yml b/roles/setup_dbt3/vars/EPAS.yml index 6618fed0e..93cfd4cc0 100644 --- a/roles/setup_dbt3/vars/EPAS.yml +++ b/roles/setup_dbt3/vars/EPAS.yml @@ -1,6 +1,3 @@ --- pg_owner: "enterprisedb" -pg_port: "5444" -pg_superuser: "enterprisedb" -pg_instance_name: "main" -pg_data: "/var/lib/edb/as{{ pg_version }}/{{ pg_instance_name }}/data" +pg_user_home: "/var/lib/edb" diff --git a/roles/setup_dbt3/vars/PG.yml b/roles/setup_dbt3/vars/PG.yml index 15ed33273..642db9a9e 100644 --- a/roles/setup_dbt3/vars/PG.yml +++ b/roles/setup_dbt3/vars/PG.yml @@ -1,6 +1,3 @@ --- pg_owner: "postgres" -pg_port: "5432" -pg_superuser: "postgres" -pg_instance_name: "main" -pg_data: "/var/lib/pgsql/{{ pg_version }}/{{ pg_instance_name }}/data" +pg_user_home: "/var/lib/pgsql" diff --git a/tests/tests/test_setup_dbt3.py b/tests/tests/test_setup_dbt3.py index 2d6317eec..fb7f540cb 100644 --- a/tests/tests/test_setup_dbt3.py +++ b/tests/tests/test_setup_dbt3.py @@ -10,7 +10,21 @@ def test_setup_dbt3_packages(): host = get_primary() - packages = ['dbt3'] + packages = [ + 'bc', + 'dejavu-fonts-common', + 'fontconfig', + 'fuse', + 'fuse-libs', + 'gcc', + 'make', + 'patch', + 'perf', + 'psmisc', + 'sysstat', + 'tmux', + 'unzip' + ] for package in packages: assert host.package(package).is_installed, \