Merge pull request #637 from panchal-yash/pxb-innovation-lts-package-…

…tests

PXB Innovation LTS setup

VERSIONS

@@ -58,3 +58,5 @@ PS_INN_LTS_REPO="ps-8x-innovation"
 PXB_INN_LTS_VER="8.2.0-1"
 PXB_INN_LTS_REV="53f63c8a"
 PXB_INN_LTS_REPO="pxb-8x-innovation"
+PXB_INN_LTS_MAJ_VER="8.2.0"
+PXB_INN_LTS_PKG_VER="1"

package_check.sh

@@ -51,9 +51,9 @@ elif [ $1 = "pxb24" ]; then
 elif [ $1 = "pxb80" ]; then
   version=${PXB80_VER}
   pkg_version=${PXB80_PKG_VER}
-elif [ $1 = "pxb81" ]; then
-  version=${PXB81_VER}
-  pkg_version=${PXB81_PKG_VER}
+elif [[ $1 =~ ^pxb8[1-9]{1}$ ]]; then
+  version=${PXB_INN_LTS_VER}
+  pkg_version=${PXB_INN_LTS_PKG_VER}
 elif [ $1 = "psmdb30" ]; then
   version=${PSMDB30_VER}
 elif [ $1 = "psmdb32" ]; then

playbooks/ps_lts_innovation.yml

@@ -56,7 +56,7 @@
     shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"'
     register: ps_inn_lts_repo_name
 
-  - name: Set major_release_version variable
+  - name: Set ps_inn_lts_repo_name variable
     set_fact:
       ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}"
 

playbooks/pxb_innovation_lts.yml

@@ -0,0 +1,156 @@
+---
+# This playbook does following:
+#   enables Percona testing repository
+#   installs latest version of PS 8x Innovation-LTS, PT and PXB 8x Innovation-LTS
+#   does some tests
+
+# Cosmic is still missing python
+# - import_playbook: test_prep.yml
+
+- hosts: all
+  become: true
+  become_method: sudo
+
+  tasks:
+  - name: include tasks for test env setup
+    include_tasks: ../tasks/test_prep.yml
+
+  - name: Extract version number using shell commands
+    shell: cat ../VERSIONS | grep -oP 'PS_INN_LTS_VER="\K(\d+)\.(\d+)' | tr -d '.'
+    register: major_release_version
+
+  - name: Set major_release_version variable
+    set_fact:
+      major_release_version: "{{ major_release_version.stdout }}"
+
+  - name: Extract values using shell command for repo name used for innovation/lts release
+    shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"'
+    register: ps_inn_lts_repo_name
+
+  - name: Set pxb_inn_lts_repo_name variable
+    set_fact:
+      ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}"
+
+  - name: include tasks for enabling PS {{ major_release_version }} main repo
+    include: ../tasks/enable_ps_innovation_repo_main.yml
+    when: lookup('env', 'install_repo') == "main"
+
+  - name: include tasks for enabling PS {{ major_release_version }} test repo
+    include: ../tasks/enable_ps_innovation_repo_testing.yml
+    when: lookup('env', 'install_repo') == "testing" or lookup('env', 'install_repo') == ""
+
+  - name: include tasks for enabling PS {{ major_release_version }} experimental repo
+    include: ../tasks/enable_ps_innovation_repo_experimental.yml
+    when: lookup('env', 'install_repo') == "experimental"
+
+  - name: download and extract world database
+    command: "{{ item }}"
+    with_items:
+    - wget --no-check-certificate -P /package-testing https://raw.githubusercontent.com/Percona-QA/percona-qa/master/sample_db/world.sql
+
+#  Disable Percona Toolkit installation till its supported in OL9 and Jammy
+#  - name: install Percona Toolkit new deb packages
+#    include_tasks: ../tasks/install_pt.yml
+
+  - name: install Percona Server {{ major_release_version }} packages
+    include_tasks: ../tasks/install_ps_innovation_lts.yml
+
+  - name: install Percona XtraBackup {{ major_release_version }} packages
+    include_tasks: ../tasks/install_pxb_innovation_lts.yml
+    when: lookup('env', 'install_repo') != "experimental"
+
+  - name: start mysql service with service command
+    command: service mysql start
+    when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6"
+
+  - name: start mysql service with systemctl command
+    command: systemctl start mysql
+    when: ansible_os_family == "RedHat" and ansible_distribution_major_version >= "7"
+
+  - name: set root password on centos
+    command: /package-testing/setpass_57.sh
+    when: ansible_os_family == "RedHat"
+
+  - name: install plugins, import world database
+    command: /package-testing/plugins_test.sh
+
+  - name: check that Percona XtraBackup version is correct
+    command: /package-testing/version_check.sh pxb{{ major_release_version }}
+
+  - name: check that Percona XtraBackup package versions are correct
+    command: /package-testing/package_check.sh pxb{{ major_release_version }}
+
+  - name: run backup
+    command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/
+
+  - name: prepare backup
+    command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/
+
+  - name: install lz4 and zstd packages on Redhat/CentOS
+    yum:
+      name: "{{ packages }}"
+    vars:
+      packages:
+      - lz4
+      - zstd
+    when: ansible_os_family == "RedHat"
+
+  - name: install lz4 and zstd packages on Ubuntu/Debian
+    apt:
+      name: "{{ packages }}"
+    vars:
+      packages:
+      - lz4
+      - zstd
+    when: ansible_os_family == "Debian" and ansible_distribution_release != "bionic"
+
+  - name: install lz4 and zstd packages on Bionic 
+    apt:
+      name: "{{ packages }}"
+    vars: 
+      packages:
+      - liblz4-tool
+      - zstd
+    when: ansible_os_family == "Debian" and ansible_distribution_release == "bionic"
+
+  - name: run backup with lz4 compression
+    command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backup_l/ --compress=lz4 --compress-threads=10
+
+  - name: decompress backup
+    command: /usr/bin/xtrabackup --decompress --user=root --target-dir=/tmp/backup_l/
+
+  - name: prepare backup
+    command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backup_l/
+
+  - name: run backup with zstd compression
+    command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backup_z/ --compress=zstd --compress-threads=10
+
+  - name: decompress backup
+    command: /usr/bin/xtrabackup --decompress --user=root --target-dir=/tmp/backup_z/
+
+  - name: prepare backup
+    command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backup_z/
+
+  - name: remove Percona Server deb packages
+    apt:
+      name: "{{ packages }}"
+      update_cache: no
+      state: absent
+    vars:
+      packages:
+      - percona-server-server*
+      - percona-xtrabackup-{{ major_release_version }}
+    when: ansible_os_family == "Debian"
+
+  - name: remove Percona Server rpm packages
+    yum:
+      name: "{{ packages }}"
+      state: absent
+    vars:
+     packages:
+      - percona-server-server*
+      - percona-xtrabackup-{{ major_release_version }}
+    when: ansible_os_family == "RedHat"
+
+  - name: check if process is stopped after package removal
+    command: /package-testing/check_running.sh mysql stopped

playbooks/pxb_kmip_innovation.yml

@@ -0,0 +1,182 @@
+---
+# This playbook does the following:
+#   enables Percona testing repository
+#   installs latest version of PyKmip, PS innovation lts, PXB innovation lts and runs some tests
+
+- hosts: all
+  become: true
+  become_method: sudo
+
+  tasks:
+  - name: include tasks for test env setup
+    include_tasks: ../tasks/test_prep.yml
+
+  - name: disable selinux for RedHat/CentOS
+    selinux: state=disabled
+    when: ansible_os_family == "RedHat"
+
+  - name: Extract version number using shell commands
+    shell: cat ../VERSIONS | grep -oP 'PS_INN_LTS_VER="\K(\d+)\.(\d+)' | tr -d '.'
+    register: major_release_version
+
+  - name: Set major_release_version variable
+    set_fact:
+      major_release_version: "{{ major_release_version.stdout }}"
+
+  - name: Extract values using shell command for repo name used for innovation/lts release
+    shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"'
+    register: ps_inn_lts_repo_name
+
+  - name: Set pxb_inn_lts_repo_name variable
+    set_fact:
+      ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}"
+
+  - name: include tasks for enabling PS {{ major_release_version }} main repo
+    include: ../tasks/enable_ps_innovation_repo_main.yml
+    when: lookup('env', 'install_repo') == "main"
+
+  - name: include tasks for enabling PS {{ major_release_version }} test repo
+    include: ../tasks/enable_ps_innovation_repo_testing.yml
+    when: lookup('env', 'install_repo') == "testing" or lookup('env', 'install_repo') == ""
+
+  - name: include tasks for enabling PS {{ major_release_version }} experimental repo
+    include: ../tasks/enable_ps_innovation_repo_experimental.yml
+    when: lookup('env', 'install_repo') == "experimental"
+
+  - name: download and extract world database
+    command: "{{ item }}"
+    with_items:
+    - wget --no-check-certificate -P /package-testing https://raw.githubusercontent.com/Percona-QA/percona-qa/master/sample_db/world.sql
+
+  - name: install Percona Server {{ major_release_version }} packages
+    include_tasks: ../tasks/install_ps_innovation_lts.yml
+
+  - name: install sysbench new deb packages
+    include_tasks: ../tasks/install_sysbench.yml
+    when: lookup('env', 'install_repo') != "experimental" and ansible_distribution_release != "bullseye"
+
+  - name: list installed packages
+    include_tasks: ../tasks/list_installed_packages.yml
+
+  - name: stop mysql service
+    service: name=mysql state=stopped
+
+  - name: remove mysql data directory
+    command: rm -r /var/lib/mysql
+
+  - name: install python3-pip on RedHat/CentOS
+    yum:
+      name:
+      - python3-pip
+      state: latest
+      update_cache: yes
+    when: ansible_os_family == "RedHat"
+
+  - name: install python3-pip on Debian/Ubuntu
+    apt:
+      name:
+      - python3-pip
+      state: latest
+      update_cache: yes
+    when: ansible_os_family == "Debian"
+
+  - name: install pykmip
+    command: "{{ item }}"
+    with_items:
+        - python3 -m pip install -U pip
+        - python3 -m pip install -U setuptools
+        - python3 -m pip install -U pykmip
+    when: ansible_distribution_release != "bookworm"
+
+  - name: install pykmip
+    command: "{{ item }}"
+    with_items:
+        - python3 -m pip install -U --break-system-packages pip
+        - python3 -m pip install -U --break-system-packages setuptools
+        - python3 -m pip install -U --break-system-packages pykmip
+    when: ansible_distribution_release == "bookworm"    
+
+  - name: downgrade SQLAlchemy as new version doesn't work with pykmip
+    command: pip3 install SQLAlchemy==1.4.46
+    when: ansible_distribution_release != "bookworm"
+
+  - name: downgrade SQLAlchemy as new version doesn't work with pykmip
+    command: pip3 install --break-system-packages SQLAlchemy==1.4.46
+    when: ansible_distribution_release == "bookworm"  
+
+  - name: start pykmip server
+    shell: nohup /usr/local/bin/pykmip-server -f /package-testing/kmip/server.conf -l /pykmip_server.log 2>&1 &
+
+  - name: copy the global manifest for mysql
+    copy:
+        src: /package-testing/kmip/mysqld.my
+        dest: /usr/sbin/
+        remote_src: yes
+
+  - name: copy the global configuration file for mysql in Redhat/CentOS
+    copy:
+        src: /package-testing/kmip/component_keyring_kmip.cnf
+        dest: /usr/lib64/mysql/plugin/
+        remote_src: yes
+    when: ansible_os_family == "RedHat"
+
+  - name: copy the global configuration file for mysql in Debian/Ubuntu
+    copy:
+        src: /package-testing/kmip/component_keyring_kmip.cnf
+        dest: /usr/lib/mysql/plugin/
+        remote_src: yes
+    when: ansible_os_family == "Debian"
+
+  - name: copy the encryption config file on Debian/Ubuntu
+    copy:
+        src: /package-testing/kmip/encryption.j2
+        dest: /etc/mysql/conf.d/encryption.cnf
+        remote_src: yes
+    when: ansible_os_family == "Debian"
+
+  - name: append include for RedHat
+    lineinfile:
+        path: /etc/my.cnf
+        line: '!includedir /etc/my.cnf.d'
+    when: ansible_os_family == "RedHat"
+
+  - name: copy the encryption config file on RHEL/CentOS/Amazon
+    copy:
+        src: /package-testing/kmip/encryption.j2
+        dest: /etc/my.cnf.d/encryption.cnf
+        remote_src: yes
+    when: ansible_os_family == "RedHat"
+
+  - name: start mysql service
+    service: name=mysql state=started
+
+  - name: check mysql service status
+    command: /package-testing/check_running.sh mysql running
+
+  - name: set root password on centos
+    command: /package-testing/setpass_57.sh
+    when: ansible_os_family == "RedHat"
+
+  - name: import world database
+    command: /package-testing/plugins_test.sh
+
+  - name: install Percona XtraBackup {{ major_release_version }} packages
+    include_tasks: ../tasks/install_pxb_innovation_lts.yml
+    when: lookup('env', 'install_repo') != "experimental"
+
+  - name: check that Percona XtraBackup version is correct
+    command: /package-testing/version_check.sh pxb{{ major_release_version }}
+
+  - name: check that Percona XtraBackup package versions are correct
+    command: /package-testing/package_check.sh pxb{{ major_release_version }}
+
+  - name: run backup
+    command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/
+
+  - name: prepare backup on Redhat/CentOS
+    command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/ --component-keyring-config=/usr/lib64/mysql/plugin/component_keyring_kmip.cnf
+    when: ansible_os_family == "RedHat"
+
+  - name: prepare backup on Debian/Ubuntu
+    command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/ --component-keyring-config=/usr/lib/mysql/plugin/component_keyring_kmip.cnf
+    when: ansible_os_family == "Debian"