Update dbt3 roles to use AppImage

roles/setup_dbt3/README.md

@@ -2,53 +2,35 @@
 
 This role is for installing DBT-3.
 
-
-
 ## Requirements
 
 Following are the requirements of this role.
   1. Ansible
-  2. `edb_devops.edb_postgres` -> `setup_repo` role for setting the repository on
-     the systems.
 
 ## Role Variables
 
 When executing the role via ansible these are the required variables:
 
-  * ***dbt3_pgdata***
-
-  This variable is to be used to specify a custom PGDATA directory that DBT-3
-  will use separately from the default PGDATA that may be used in other
-  playbooks.  Thus do not set this to a directory that may be used elsewhere.
-  You are not likely to use this unless you are testing specific source based
-  installations of PostgreSQL or doing any other highly customized testing with
-  logical or physical system configurations.
-
   * ***dbt3_version***
 
-  These playbooks can install any version of DBT-3 that is packaged from GitHub.
-  See the following link for available versions:
-  https://github.com/osdldbt/dbt3-packaging/releases
+  These playbooks can install any version of the DBT-3 AppImage that is
+  packaged from GitHub.  See the following link for available versions:
+  https://github.com/osdldbt/dbt3/releases
 
   * ***have_tpcfile***
 
-  The is a boolean use for CI purposes where we cannot redistribute TPC code
+  This is a boolean used for CI purposes where we cannot redistribute TPC code
   thus the CI system will not attempt to run the tasks specific for setting up
   the TPC code.
 
-  * ***pg_version***
-
-  Postgres Versions supported are: 10, 11, 12, 13 and 14
-
   * ***tpcfile***
 
   This is the full path to the TPC-H Tools zip file that must be downloaded by
-  the you, the user, per TPC EULA agreement.  The TPC-H Tools can be requested
-  at:
+  you, the user, per TPC EULA agreement.  The TPC-H Tools can be requested at:
   https://www.tpc.org/tpc_documents_current_versions/current_specifications5.asp
 
 These variables can be assigned in the `pre_tasks` definition of the
-section: *How to include the `setup_dbt3` role in your Playbook*.
+section.
 
 ## Example Playbook
 
@@ -86,16 +68,8 @@ Below is an example of how to include the `setup_dbt3` role:
     - edb_devops.edb_postgres
 
   roles:
-    - role: setup_repo
-      when: "'setup_repo' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)"
-    - role: install_dbserver
-      when: "'install_dbserver' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)"
-    - role: init_dbserver
-      when: "'init_dbserver' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)"
     - role: setup_dbt3
       when: "'setup_dbt3' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)"
-    - role: tuning
-      when: "'tuning' in lookup('edb_devops.edb_postgres.supported_roles', wantlist=True)"
 ```
 
 ## Playbook execution examples
@@ -106,7 +80,7 @@ $ ansible-playbook playbook.yml \
   -i inventory.yml \
   -u centos \
   --private-key <key.pem> \
-  --extra-vars="pg_version=13 pg_type=PG"
+  --extra-vars="pg_type=PG"
 ```
 
 ## License

roles/setup_dbt3/defaults/main.yml

@@ -1,10 +1,13 @@
 ---
-dbt3_version: "1.11.0"
-dbt3_pgdata: "/pgdata/dbt3/pgdata"
-pg_version: 14
+dbt3_version: "2.0.1"
 have_tpcfile: true
 tpcfile: ""
 
+# DBT-3 AppImage information
+dbt3_appimage: "https://github.com/osdldbt/dbt3/releases/download/v{{ dbt3_version }}/dbt3-{{ dbt3_version }}-x86_64.AppImage"
+dbt3_checksum: "{{ dbt3_appimage }}.sha256"
+dbt3_path: "/usr/bin/dbt3"
+
 # setting validate_only to true allows you to validate setup on an existing node
 # use_validation flag applies to deployment configuration and validation after setup
 validate_only: false

roles/setup_dbt3/tasks/dbt3_install_packages.yml

@@ -3,23 +3,26 @@
   ansible.builtin.package:
     name:
       - bc
+      - dejavu-fonts-common
+      - fontconfig
+      - fuse
+      - fuse-libs
       - gcc
+      - make
       - patch
       - perf
       - psmisc
-      - python3-docutils
-      - R-core
       - sysstat
       - tmux
+      - unzip
     state: present
   when: ansible_os_family == "RedHat"
   become: true
 
-- name: Install packages for DBT-3 Database
-  ansible.builtin.dnf:
-    name:
-      - 'https://github.com/osdldbt/dbt3-packaging/releases/download/v{{ dbt3_version }}/dbt3-{{ dbt3_version }}-1.el8.x86_64.rpm'
-    state: present
-    disable_gpg_check: true
-  when: ansible_os_family == "RedHat"
+- name: Download DBT-3 AppImage
+  ansible.builtin.get_url:
+    url: "{{ dbt3_appimage }}"
+    checksum: "sha256:{{ dbt3_checksum }}"
+    dest: "{{ dbt3_path }}"
+    mode: '0755'
   become: true

roles/setup_dbt3/tasks/dbt3_tpc.yml

@@ -13,19 +13,7 @@
     dest: "{{ pg_user_home }}"
   become_user: "{{ pg_owner }}"
 
-- name: Patch TPC-H Tools for PostgreSQL
-  ansible.posix.patch:
-    remote_src: true
-    src: "{{ item }}"
-    basedir: "{{ pg_user_home }}"
-    strip: 1
-  become_user: "{{ pg_owner }}"
-  loop:
-    - /usr/share/dbt3/dbt3-TPC-H_Tools_v3.0.0-postgresql.diff
-    - /usr/share/dbt3/dbt3-TPC-H_Tools_v3.0.0-stdout.diff
-
 - name: Build TPC-H Tools for PostgreSQL
-  community.general.make:
-    chdir: "{{ pg_user_home }}/TPC-H_Tools_v3.0.0/dbgen"
-    file: Makefile.postgresql
+  ansible.builtin.shell: >-
+    dbt3 build-dbgen pgsql {{ pg_user_home }}/'TPC-H V3.0.1'
   become_user: "{{ pg_owner }}"

roles/setup_dbt3/tasks/main.yml

@@ -10,10 +10,6 @@
   ansible.builtin.include_tasks: dbt3_tpc.yml
   when: have_tpcfile and not validate_only|bool
 
-- name: Include DBT-3 profile configuration
-  ansible.builtin.include_tasks: dbt3_configure_profile.yml
-  when: not validate_only|bool
-
 - name: Include the validate_setup_dbt3 tasks
   ansible.builtin.include_tasks: validate_setup_dbt3.yml
   when: validate_only|bool or use_validation|bool

roles/setup_dbt3/tasks/validate_setup_dbt3.yml

@@ -1,45 +1,40 @@
 ---
-# validate dbt3 packages
-- name: Set dbt3 package name
+# validate dbt3 db packages installation
+- name: Set dbt3_db_pkg_list
   ansible.builtin.set_fact:
-    dbt3_package: 'dbt3'
+    dbt3_db_pkg_list: [ 'bc', 'dejavu-fonts-common', 'fontconfig', 'fuse', 'fuse-libs', 'gcc', 'make', 'patch', 'perf', 'psmisc', 'sysstat', 'tmux', 'unzip' ]
 
 - name: Gather the package facts
   ansible.builtin.package_facts:
     strategy: "all"
 
-- name: Check that DBT3 packages have been installed
+- name: Check that all packages in dbt3_db_pkg_list has been installed
   ansible.builtin.assert:
     that:
-      - "{{ ansible_facts.packages[dbt3_package] }} is defined"
-    fail_msg: "Package {{ dbt3_package }} has not been installed."
-    success_msg: "Package {{ dbt3_package }} has been installed."
+      - "{{ ansible_facts.packages[package] }} is defined"
+    fail_msg: "Package {{ package }} has not been installed."
+    success_msg: "Package {{ package }} has been installed."
+  loop: "{{ dbt3_db_pkg_list }}"
+  loop_control:
+    loop_var: package
 
-# validate dbt3 user
-- name: Run query to check dbt3 user is in pg_user
-  ansible.builtin.include_role:
-    name: manage_dbserver
-    tasks_from: execute_sql_scripts
-  vars:
-    pg_query:
-      - query: "Select * from pg_user where usename = '{{ pg_owner }}'"
-        db: "{{ pg_database }}"
-
-- name: Set user_query_result with sql_query_output
-  ansible.builtin.set_fact:
-    user_query_result: "{{ sql_query_output }}"
+# ensure the DBT-3 AppImage has been installed to the proper location
+- name: Stat to dbt3_path
+  ansible.builtin.stat:
+    path: "{{ dbt3_path }}"
+  register: dbt3_path_stat
   become: true
 
-- name: Check if DBT3 postgres user was successfully created
+- name: Check if DBT-3 AppImage was installed properly on DB
   ansible.builtin.assert:
     that:
-      - user_query_result.results[0].query_result[0]['usename'] == pg_owner
-    fail_msg: "DBT3 postgres user was not successfully created."
-    success_msg: "DBT3 postgres user was successfully created."
-  run_once: true
+      - dbt3_path_stat.stat['exists']|bool
+    fail_msg: "DBT-3 AppImage at {{ dbt3_path }} was not created properly"
+    success_msg: "DBT-3 AppImage at {{ dbt3_path }} was created successfully"
 
 # reset variables
 - name: Reset variables
   ansible.builtin.set_fact:
-    dbt3_package: null
+    dbt3_db_pkg_list: null
+    dbt3_path_stat: null
     user_query_result: null

roles/setup_dbt3/vars/EPAS.yml

@@ -1,6 +1,3 @@
 ---
 pg_owner: "enterprisedb"
-pg_port: "5444"
-pg_superuser: "enterprisedb"
-pg_instance_name: "main"
-pg_data: "/var/lib/edb/as{{ pg_version }}/{{ pg_instance_name }}/data"
+pg_user_home: "/var/lib/edb"

roles/setup_dbt3/vars/PG.yml

@@ -1,6 +1,3 @@
 ---
 pg_owner: "postgres"
-pg_port: "5432"
-pg_superuser: "postgres"
-pg_instance_name: "main"
-pg_data: "/var/lib/pgsql/{{ pg_version }}/{{ pg_instance_name }}/data"
+pg_user_home: "/var/lib/pgsql"

tests/tests/test_setup_dbt3.py

@@ -10,7 +10,21 @@
 
 def test_setup_dbt3_packages():
     host = get_primary()
-    packages = ['dbt3']
+    packages = [
+        'bc',
+        'dejavu-fonts-common',
+        'fontconfig',
+        'fuse',
+        'fuse-libs',
+        'gcc',
+        'make',
+        'patch',
+        'perf',
+        'psmisc',
+        'sysstat',
+        'tmux',
+        'unzip'
+    ]
 
     for package in packages:
         assert host.package(package).is_installed, \