From 353ca8f237287721e9566235dd74f2021d0ff05b Mon Sep 17 00:00:00 2001 From: yash Date: Thu, 1 Feb 2024 18:25:24 +0530 Subject: [PATCH] PXB Innovation LTS setup Use tarballs for upstream tests modify pxb kms innovation playbook --- VERSIONS | 2 + package_check.sh | 6 +- playbooks/ps_lts_innovation.yml | 2 +- playbooks/pxb_innovation_lts.yml | 156 +++++++++++++++ playbooks/pxb_kmip_innovation.yml | 182 +++++++++++++++++ playbooks/pxb_kms_innovation.yml | 50 +++++ playbooks/pxb_tarball_innovation_lts.yml | 145 ++++++++++++++ playbooks/pxb_upgrade_innovation_lts.yml | 152 ++++++++++++++ playbooks/pxb_upstream_innovation_lts.yml | 229 ++++++++++++++++++++++ test_binary_tarball.sh | 18 +- 10 files changed, 931 insertions(+), 11 deletions(-) create mode 100644 playbooks/pxb_innovation_lts.yml create mode 100644 playbooks/pxb_kmip_innovation.yml create mode 100644 playbooks/pxb_kms_innovation.yml create mode 100644 playbooks/pxb_tarball_innovation_lts.yml create mode 100644 playbooks/pxb_upgrade_innovation_lts.yml create mode 100644 playbooks/pxb_upstream_innovation_lts.yml diff --git a/VERSIONS b/VERSIONS index 1ae95b3b9..5d5ee883b 100644 --- a/VERSIONS +++ b/VERSIONS @@ -58,3 +58,5 @@ PS_INN_LTS_REPO="ps-8x-innovation" PXB_INN_LTS_VER="8.2.0-1" PXB_INN_LTS_REV="53f63c8a" PXB_INN_LTS_REPO="pxb-8x-innovation" +PXB_INN_LTS_MAJ_VER="8.2.0" +PXB_INN_LTS_PKG_VER="1" diff --git a/package_check.sh b/package_check.sh index 3b3329478..126de6166 100755 --- a/package_check.sh +++ b/package_check.sh @@ -51,9 +51,9 @@ elif [ $1 = "pxb24" ]; then elif [ $1 = "pxb80" ]; then version=${PXB80_VER} pkg_version=${PXB80_PKG_VER} -elif [ $1 = "pxb81" ]; then - version=${PXB81_VER} - pkg_version=${PXB81_PKG_VER} +elif [[ $1 =~ ^pxb8[1-9]{1}$ ]]; then + version=${PXB_INN_LTS_VER} + pkg_version=${PXB_INN_LTS_PKG_VER} elif [ $1 = "psmdb30" ]; then version=${PSMDB30_VER} elif [ $1 = "psmdb32" ]; then diff --git a/playbooks/ps_lts_innovation.yml b/playbooks/ps_lts_innovation.yml index b63ce32ae..dda7bc683 100644 --- a/playbooks/ps_lts_innovation.yml +++ b/playbooks/ps_lts_innovation.yml @@ -56,7 +56,7 @@ shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"' register: ps_inn_lts_repo_name - - name: Set major_release_version variable + - name: Set ps_inn_lts_repo_name variable set_fact: ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}" diff --git a/playbooks/pxb_innovation_lts.yml b/playbooks/pxb_innovation_lts.yml new file mode 100644 index 000000000..7cf8ca990 --- /dev/null +++ b/playbooks/pxb_innovation_lts.yml @@ -0,0 +1,156 @@ +--- +# This playbook does following: +# enables Percona testing repository +# installs latest version of PS 8x Innovation-LTS, PT and PXB 8x Innovation-LTS +# does some tests + +# Cosmic is still missing python +# - import_playbook: test_prep.yml + +- hosts: all + become: true + become_method: sudo + + tasks: + - name: include tasks for test env setup + include_tasks: ../tasks/test_prep.yml + + - name: Extract version number using shell commands + shell: cat ../VERSIONS | grep -oP 'PS_INN_LTS_VER="\K(\d+)\.(\d+)' | tr -d '.' + register: major_release_version + + - name: Set major_release_version variable + set_fact: + major_release_version: "{{ major_release_version.stdout }}" + + - name: Extract values using shell command for repo name used for innovation/lts release + shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"' + register: ps_inn_lts_repo_name + + - name: Set pxb_inn_lts_repo_name variable + set_fact: + ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}" + + - name: include tasks for enabling PS {{ major_release_version }} main repo + include: ../tasks/enable_ps_innovation_repo_main.yml + when: lookup('env', 'install_repo') == "main" + + - name: include tasks for enabling PS {{ major_release_version }} test repo + include: ../tasks/enable_ps_innovation_repo_testing.yml + when: lookup('env', 'install_repo') == "testing" or lookup('env', 'install_repo') == "" + + - name: include tasks for enabling PS {{ major_release_version }} experimental repo + include: ../tasks/enable_ps_innovation_repo_experimental.yml + when: lookup('env', 'install_repo') == "experimental" + + - name: download and extract world database + command: "{{ item }}" + with_items: + - wget --no-check-certificate -P /package-testing https://raw.githubusercontent.com/Percona-QA/percona-qa/master/sample_db/world.sql + +# Disable Percona Toolkit installation till its supported in OL9 and Jammy +# - name: install Percona Toolkit new deb packages +# include_tasks: ../tasks/install_pt.yml + + - name: install Percona Server {{ major_release_version }} packages + include_tasks: ../tasks/install_ps_innovation_lts.yml + + - name: install Percona XtraBackup {{ major_release_version }} packages + include_tasks: ../tasks/install_pxb_innovation_lts.yml + when: lookup('env', 'install_repo') != "experimental" + + - name: start mysql service with service command + command: service mysql start + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6" + + - name: start mysql service with systemctl command + command: systemctl start mysql + when: ansible_os_family == "RedHat" and ansible_distribution_major_version >= "7" + + - name: set root password on centos + command: /package-testing/setpass_57.sh + when: ansible_os_family == "RedHat" + + - name: install plugins, import world database + command: /package-testing/plugins_test.sh + + - name: check that Percona XtraBackup version is correct + command: /package-testing/version_check.sh pxb{{ major_release_version }} + + - name: check that Percona XtraBackup package versions are correct + command: /package-testing/package_check.sh pxb{{ major_release_version }} + + - name: run backup + command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/ + + - name: prepare backup + command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/ + + - name: install lz4 and zstd packages on Redhat/CentOS + yum: + name: "{{ packages }}" + vars: + packages: + - lz4 + - zstd + when: ansible_os_family == "RedHat" + + - name: install lz4 and zstd packages on Ubuntu/Debian + apt: + name: "{{ packages }}" + vars: + packages: + - lz4 + - zstd + when: ansible_os_family == "Debian" and ansible_distribution_release != "bionic" + + - name: install lz4 and zstd packages on Bionic + apt: + name: "{{ packages }}" + vars: + packages: + - liblz4-tool + - zstd + when: ansible_os_family == "Debian" and ansible_distribution_release == "bionic" + + - name: run backup with lz4 compression + command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backup_l/ --compress=lz4 --compress-threads=10 + + - name: decompress backup + command: /usr/bin/xtrabackup --decompress --user=root --target-dir=/tmp/backup_l/ + + - name: prepare backup + command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backup_l/ + + - name: run backup with zstd compression + command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backup_z/ --compress=zstd --compress-threads=10 + + - name: decompress backup + command: /usr/bin/xtrabackup --decompress --user=root --target-dir=/tmp/backup_z/ + + - name: prepare backup + command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backup_z/ + + - name: remove Percona Server deb packages + apt: + name: "{{ packages }}" + update_cache: no + state: absent + vars: + packages: + - percona-server-server* + - percona-xtrabackup-{{ major_release_version }} + when: ansible_os_family == "Debian" + + - name: remove Percona Server rpm packages + yum: + name: "{{ packages }}" + state: absent + vars: + packages: + - percona-server-server* + - percona-xtrabackup-{{ major_release_version }} + when: ansible_os_family == "RedHat" + + - name: check if process is stopped after package removal + command: /package-testing/check_running.sh mysql stopped diff --git a/playbooks/pxb_kmip_innovation.yml b/playbooks/pxb_kmip_innovation.yml new file mode 100644 index 000000000..662fd021a --- /dev/null +++ b/playbooks/pxb_kmip_innovation.yml @@ -0,0 +1,182 @@ +--- +# This playbook does the following: +# enables Percona testing repository +# installs latest version of PyKmip, PS innovation lts, PXB innovation lts and runs some tests + +- hosts: all + become: true + become_method: sudo + + tasks: + - name: include tasks for test env setup + include_tasks: ../tasks/test_prep.yml + + - name: disable selinux for RedHat/CentOS + selinux: state=disabled + when: ansible_os_family == "RedHat" + + - name: Extract version number using shell commands + shell: cat ../VERSIONS | grep -oP 'PS_INN_LTS_VER="\K(\d+)\.(\d+)' | tr -d '.' + register: major_release_version + + - name: Set major_release_version variable + set_fact: + major_release_version: "{{ major_release_version.stdout }}" + + - name: Extract values using shell command for repo name used for innovation/lts release + shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"' + register: ps_inn_lts_repo_name + + - name: Set pxb_inn_lts_repo_name variable + set_fact: + ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}" + + - name: include tasks for enabling PS {{ major_release_version }} main repo + include: ../tasks/enable_ps_innovation_repo_main.yml + when: lookup('env', 'install_repo') == "main" + + - name: include tasks for enabling PS {{ major_release_version }} test repo + include: ../tasks/enable_ps_innovation_repo_testing.yml + when: lookup('env', 'install_repo') == "testing" or lookup('env', 'install_repo') == "" + + - name: include tasks for enabling PS {{ major_release_version }} experimental repo + include: ../tasks/enable_ps_innovation_repo_experimental.yml + when: lookup('env', 'install_repo') == "experimental" + + - name: download and extract world database + command: "{{ item }}" + with_items: + - wget --no-check-certificate -P /package-testing https://raw.githubusercontent.com/Percona-QA/percona-qa/master/sample_db/world.sql + + - name: install Percona Server {{ major_release_version }} packages + include_tasks: ../tasks/install_ps_innovation_lts.yml + + - name: install sysbench new deb packages + include_tasks: ../tasks/install_sysbench.yml + when: lookup('env', 'install_repo') != "experimental" and ansible_distribution_release != "bullseye" + + - name: list installed packages + include_tasks: ../tasks/list_installed_packages.yml + + - name: stop mysql service + service: name=mysql state=stopped + + - name: remove mysql data directory + command: rm -r /var/lib/mysql + + - name: install python3-pip on RedHat/CentOS + yum: + name: + - python3-pip + state: latest + update_cache: yes + when: ansible_os_family == "RedHat" + + - name: install python3-pip on Debian/Ubuntu + apt: + name: + - python3-pip + state: latest + update_cache: yes + when: ansible_os_family == "Debian" + + - name: install pykmip + command: "{{ item }}" + with_items: + - python3 -m pip install -U pip + - python3 -m pip install -U setuptools + - python3 -m pip install -U pykmip + when: ansible_distribution_release != "bookworm" + + - name: install pykmip + command: "{{ item }}" + with_items: + - python3 -m pip install -U --break-system-packages pip + - python3 -m pip install -U --break-system-packages setuptools + - python3 -m pip install -U --break-system-packages pykmip + when: ansible_distribution_release == "bookworm" + + - name: downgrade SQLAlchemy as new version doesn't work with pykmip + command: pip3 install SQLAlchemy==1.4.46 + when: ansible_distribution_release != "bookworm" + + - name: downgrade SQLAlchemy as new version doesn't work with pykmip + command: pip3 install --break-system-packages SQLAlchemy==1.4.46 + when: ansible_distribution_release == "bookworm" + + - name: start pykmip server + shell: nohup /usr/local/bin/pykmip-server -f /package-testing/kmip/server.conf -l /pykmip_server.log 2>&1 & + + - name: copy the global manifest for mysql + copy: + src: /package-testing/kmip/mysqld.my + dest: /usr/sbin/ + remote_src: yes + + - name: copy the global configuration file for mysql in Redhat/CentOS + copy: + src: /package-testing/kmip/component_keyring_kmip.cnf + dest: /usr/lib64/mysql/plugin/ + remote_src: yes + when: ansible_os_family == "RedHat" + + - name: copy the global configuration file for mysql in Debian/Ubuntu + copy: + src: /package-testing/kmip/component_keyring_kmip.cnf + dest: /usr/lib/mysql/plugin/ + remote_src: yes + when: ansible_os_family == "Debian" + + - name: copy the encryption config file on Debian/Ubuntu + copy: + src: /package-testing/kmip/encryption.j2 + dest: /etc/mysql/conf.d/encryption.cnf + remote_src: yes + when: ansible_os_family == "Debian" + + - name: append include for RedHat + lineinfile: + path: /etc/my.cnf + line: '!includedir /etc/my.cnf.d' + when: ansible_os_family == "RedHat" + + - name: copy the encryption config file on RHEL/CentOS/Amazon + copy: + src: /package-testing/kmip/encryption.j2 + dest: /etc/my.cnf.d/encryption.cnf + remote_src: yes + when: ansible_os_family == "RedHat" + + - name: start mysql service + service: name=mysql state=started + + - name: check mysql service status + command: /package-testing/check_running.sh mysql running + + - name: set root password on centos + command: /package-testing/setpass_57.sh + when: ansible_os_family == "RedHat" + + - name: import world database + command: /package-testing/plugins_test.sh + + - name: install Percona XtraBackup {{ major_release_version }} packages + include_tasks: ../tasks/install_pxb_innovation_lts.yml + when: lookup('env', 'install_repo') != "experimental" + + - name: check that Percona XtraBackup version is correct + command: /package-testing/version_check.sh pxb{{ major_release_version }} + + - name: check that Percona XtraBackup package versions are correct + command: /package-testing/package_check.sh pxb{{ major_release_version }} + + - name: run backup + command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/ + + - name: prepare backup on Redhat/CentOS + command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/ --component-keyring-config=/usr/lib64/mysql/plugin/component_keyring_kmip.cnf + when: ansible_os_family == "RedHat" + + - name: prepare backup on Debian/Ubuntu + command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/ --component-keyring-config=/usr/lib/mysql/plugin/component_keyring_kmip.cnf + when: ansible_os_family == "Debian" diff --git a/playbooks/pxb_kms_innovation.yml b/playbooks/pxb_kms_innovation.yml new file mode 100644 index 000000000..c1a1010c8 --- /dev/null +++ b/playbooks/pxb_kms_innovation.yml @@ -0,0 +1,50 @@ +--- +# This playbook does the following: +# requires the environment vars to be set: KMS_KEY_ID, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +# enables Percona testing repository +# installs latest version of PS innovation lts, PXB innovation lts with kms component and runs some tests + +- import_playbook: ps_lts_innovation_kms.yml + +- hosts: all + become: true + become_method: sudo + + tasks: + + - name: Extract version number using shell commands + shell: cat ../VERSIONS | grep -oP 'PS_INN_LTS_VER="\K(\d+)\.(\d+)' | tr -d '.' + register: major_release_version + + - name: Set major_release_version variable + set_fact: + major_release_version: "{{ major_release_version.stdout }}" + + - name: Extract values using shell command for repo name used for innovation/lts release + shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"' + register: ps_inn_lts_repo_name + + - name: Set pxb_inn_lts_repo_name variable + set_fact: + ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}" + + - name: install Percona XtraBackup {{ major_release_version }} packages + include_tasks: ../tasks/install_pxb_innovation_lts.yml + when: lookup('env', 'install_repo') != "experimental" + + - name: check that Percona XtraBackup version is correct + command: /package-testing/version_check.sh pxb{{ major_release_version }} + + - name: check that Percona XtraBackup package versions are correct + command: /package-testing/package_check.sh pxb{{ major_release_version }} + + - name: run backup + command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/ + + - name: prepare backup on Redhat/CentOS + command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/ --component-keyring-config=/usr/lib64/mysql/plugin/component_keyring_kms.cnf + when: ansible_os_family == "RedHat" + + - name: prepare backup on Debian/Ubuntu + command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/ --component-keyring-config=/usr/lib/mysql/plugin/component_keyring_kms.cnf + when: ansible_os_family == "Debian" diff --git a/playbooks/pxb_tarball_innovation_lts.yml b/playbooks/pxb_tarball_innovation_lts.yml new file mode 100644 index 000000000..da6684acb --- /dev/null +++ b/playbooks/pxb_tarball_innovation_lts.yml @@ -0,0 +1,145 @@ +--- +# This playbook does following: +# enables Percona testing repository +# installs latest version of PS innovation lts, PXB innovation lts tarball +# then runs some tests + +- hosts: all + become: true + become_method: sudo + vars: +# binary_type: "{{ lookup('env', 'binary_type') }}" + install_repo: "{{ lookup('env', 'install_repo') }}" + + tasks: + - name: include tasks for test env setup + include_tasks: ../tasks/test_prep.yml + + - name: install libev apt package + apt: + name: "{{ packages }}" + update_cache: yes + state: latest + vars: + packages: + - libev4 + when: ansible_os_family == "Debian" + + - name: install libev rpm package + yum: + name: "{{ packages }}" + state: latest + vars: + packages: + - libev + when: ansible_os_family == "RedHat" + + - name: Extract version number using shell commands + shell: cat ../VERSIONS | grep -oP 'PS_INN_LTS_VER="\K(\d+)\.(\d+)' | tr -d '.' + register: major_release_version + + - name: Set major_release_version variable + set_fact: + major_release_version: "{{ major_release_version.stdout }}" + + - name: Extract values using shell command for repo name used for innovation/lts release + shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"' + register: ps_inn_lts_repo_name + + - name: Set pxb_inn_lts_repo_name variable + set_fact: + ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}" + + - name: include tasks for enabling PS {{ major_release_version }} main repo + include: ../tasks/enable_ps_innovation_repo_main.yml + when: lookup('env', 'install_repo') == "main" + + - name: include tasks for enabling PS {{ major_release_version }} test repo + include: ../tasks/enable_ps_innovation_repo_testing.yml + when: lookup('env', 'install_repo') == "testing" or lookup('env', 'install_repo') == "" + + - name: include tasks for enabling PS {{ major_release_version }} experimental repo + include: ../tasks/enable_ps_innovation_repo_experimental.yml + when: lookup('env', 'install_repo') == "experimental" + + - name: download and extract world database + command: "{{ item }}" + with_items: + - wget --no-check-certificate -P /package-testing https://raw.githubusercontent.com/Percona-QA/percona-qa/master/sample_db/world.sql + + - name: install Percona Server {{ major_release_version }} packages + include_tasks: ../tasks/install_ps_innovation_lts.yml + + - name: install and check Percona XtraBackup {{ major_release_version }} normal tarball from {{ install_repo }} downloads area + command: /package-testing/test_binary_tarball.sh pxb{{ major_release_version }} {{ install_repo }} normal + + - name: start mysql service with service command + command: service mysql start + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6" + + - name: start mysql service with systemctl command + command: systemctl start mysql + when: ansible_os_family == "RedHat" and ansible_distribution_major_version >= "7" + + - name: set root password on centos + command: /package-testing/setpass_57.sh + when: ansible_os_family == "RedHat" + + - name: install plugins, import world database + command: /package-testing/plugins_test.sh + + - name: run backup for Redhat/Centos + command: pxb{{ major_release_version }}/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/ --socket=/var/lib/mysql/mysql.sock + when: ansible_os_family == "RedHat" + + - name: run backup for Debian + command: pxb{{ major_release_version }}/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/ + when: ansible_os_family == "Debian" + + - name: prepare backup + command: pxb{{ major_release_version }}/bin/xtrabackup --prepare --target-dir=/tmp/backups/ + + - name: run backup for Redhat/Centos with streaming and encryption enabled + shell: pxb{{ major_release_version }}/bin/xtrabackup --backup --user=root --target-dir=backups/ --socket=/var/lib/mysql/mysql.sock --encrypt=AES256 --encrypt-key=mAU3Xs5sRcSB7zBEJP1BInPP5lgShKly --stream=xbstream --parallel=10 > backup.stream + when: ansible_os_family == "RedHat" + + - name: run backup for Debian with streaming and encryption enabled + shell: pxb{{ major_release_version }}/bin/xtrabackup --backup --user=root --target-dir=backups/ --encrypt=AES256 --encrypt-key=mAU3Xs5sRcSB7zBEJP1BInPP5lgShKly --stream=xbstream --parallel=10 > backup.stream + when: ansible_os_family == "Debian" + + - name: extract and decrypt backup + shell: pxb{{ major_release_version }}/bin/xbstream --directory=backups/ --extract --decrypt=AES256 --encrypt-key=mAU3Xs5sRcSB7zBEJP1BInPP5lgShKly < backup.stream + + - name: prepare backup + command: pxb{{ major_release_version }}/bin/xtrabackup --prepare --target-dir=backups/ + + - name: remove backup files + command: rm -r pxb{{ major_release_version }} backups /tmp/backups + + - name: install and check Percona XtraBackup {{ major_release_version }} minimal tarball from {{ install_repo }} downloads area + command: /package-testing/test_binary_tarball.sh pxb{{ major_release_version }} {{ install_repo }} minimal + + - name: run backup for Redhat/Centos + command: pxb{{ major_release_version }}/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/ --socket=/var/lib/mysql/mysql.sock + when: ansible_os_family == "RedHat" + + - name: run backup for Debian + command: pxb{{ major_release_version }}/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/ + when: ansible_os_family == "Debian" + + - name: prepare backup + command: pxb{{ major_release_version }}/bin/xtrabackup --prepare --target-dir=/tmp/backups/ + + - name: run backup for Redhat/Centos with streaming and encryption enabled + shell: pxb{{ major_release_version }}/bin/xtrabackup --backup --user=root --target-dir=backups/ --socket=/var/lib/mysql/mysql.sock --encrypt=AES256 --encrypt-key=mAU3Xs5sRcSB7zBEJP1BInPP5lgShKly --stream=xbstream --parallel=10 > backup.stream + when: ansible_os_family == "RedHat" + + - name: run backup for Debian with streaming and encryption enabled + shell: pxb{{ major_release_version }}/bin/xtrabackup --backup --user=root --target-dir=backups/ --encrypt=AES256 --encrypt-key=mAU3Xs5sRcSB7zBEJP1BInPP5lgShKly --stream=xbstream --parallel=10 > backup.stream + when: ansible_os_family == "Debian" + + - name: extract and decrypt backup + shell: pxb{{ major_release_version }}/bin/xbstream --directory=backups/ --extract --decrypt=AES256 --encrypt-key=mAU3Xs5sRcSB7zBEJP1BInPP5lgShKly < backup.stream + + - name: prepare backup + command: pxb{{ major_release_version }}/bin/xtrabackup --prepare --target-dir=backups/ diff --git a/playbooks/pxb_upgrade_innovation_lts.yml b/playbooks/pxb_upgrade_innovation_lts.yml new file mode 100644 index 000000000..7617f86b3 --- /dev/null +++ b/playbooks/pxb_upgrade_innovation_lts.yml @@ -0,0 +1,152 @@ +--- +# This playbook does following: +# sets main Percona repo +# installs PS innovation lts, PXB innovation lts, PT +# changes repo to testing +# upgrades packages to the new version +# does some tests +# + +# Cosmic is still missing python +# - import_playbook: test_prep.yml + +- hosts: all + become: true + become_method: sudo + + tasks: + - name: include tasks for test env setup + include: ../tasks/test_prep.yml + +# Install from testing repository + +# - name: include tasks for enabling main repo +# include: ../tasks/enable_main_repo.yml + + - name: Extract version number using shell commands + shell: cat ../VERSIONS | grep -oP 'PS_INN_LTS_VER="\K(\d+)\.(\d+)' | tr -d '.' + register: major_release_version + + - name: Set major_release_version variable + set_fact: + major_release_version: "{{ major_release_version.stdout }}" + + - name: Extract values using shell command for repo name used for innovation/lts release + shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"' + register: ps_inn_lts_repo_name + + - name: Set pxb_inn_lts_repo_name variable + set_fact: + ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}" + + - name: include tasks for enabling PS {{ major_release_version }} main repo + include: ../tasks/enable_ps_innovation_repo_main.yml + + - name: download and extract world database + command: "{{ item }}" + with_items: + - wget --no-check-certificate -P /package-testing https://raw.githubusercontent.com/Percona-QA/percona-qa/master/sample_db/world.sql + +# Disable Percona Toolkit installation till its supported in OL9 and Jammy +# - name: install Percona Toolkit new deb packages +# include_tasks: ../tasks/install_pt.yml + + - name: install Percona Server {{ major_release_version }} packages + include_tasks: ../tasks/install_ps_innovation_lts.yml + + - name: install Percona XtraBackup {{ major_release_version }} packages + include_tasks: ../tasks/install_pxb_innovation_lts.yml + when: lookup('env', 'install_repo') != "experimental" + + - name: start mysql service with service command + command: service mysql start + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6" + + - name: start mysql service with systemctl command + command: systemctl start mysql + when: ansible_os_family == "RedHat" and ansible_distribution_major_version >= "7" + + - name: set root password on centos + command: /package-testing/setpass_57.sh + when: ansible_os_family == "RedHat" + + - name: copy .my.cnf with credentials on centos + template: src=../templates/my_57.j2 + dest=/root/.my.cnf + mode=0640 owner=root group=root + when: ansible_os_family == "RedHat" + + - name: restart mysql service + service: name=mysql state=restarted + + - name: install plugins, import world database + command: /package-testing/plugins_test.sh + + - name: install 3rd party packages with apt + apt: + name: "{{ packages }}" + vars: + packages: + - rsyslog-mysql + when: ansible_os_family == "Debian" + + - name: install 3rd party packages with yum + yum: + name: "{{ packages }}" + vars: + packages: + - rsyslog-mysql + when: ansible_os_family == "RedHat" + +# +# Enable testing repository +# + - name: include tasks for enabling PS {{ major_release_version }} test repo + include: ../tasks/enable_ps_innovation_repo_testing.yml + when: lookup('env', 'install_repo') == "testing" or lookup('env', 'install_repo') == "" + +# +# Upgrade packages +# + - name: install Percona Server {{ major_release_version }} packages + include_tasks: ../tasks/install_ps_innovation_lts.yml + + - name: install Percona XtraBackup {{ major_release_version }} packages + include_tasks: ../tasks/install_pxb_innovation_lts.yml + when: lookup('env', 'install_repo') != "experimental" + + - name: check that Percona XtraBackup version is correct + command: /package-testing/version_check.sh pxb{{ major_release_version }} + + - name: check that Percona XtraBackup package versions are correct + command: /package-testing/package_check.sh pxb{{ major_release_version }} + + - name: run backup + command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/ + + - name: prepare backup + command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/ + + - name: remove Percona Server deb packages + apt: + name: "{{ packages }}" + update_cache: no + state: absent + vars: + packages: + - percona-server-server + - percona-xtrabackup-8{{ major_release_version }} + when: ansible_os_family == "Debian" + + - name: remove Percona Server rpm packages + yum: + name: "{{ packages }}" + state: absent + vars: + packages: + - percona-server-server + - percona-xtrabackup-8{{ major_release_version }} + when: ansible_os_family == "RedHat" + + - name: check if process is stopped after package removal + command: /package-testing/check_running.sh mysql stopped diff --git a/playbooks/pxb_upstream_innovation_lts.yml b/playbooks/pxb_upstream_innovation_lts.yml new file mode 100644 index 000000000..3ca351243 --- /dev/null +++ b/playbooks/pxb_upstream_innovation_lts.yml @@ -0,0 +1,229 @@ +--- +# This playbook sets up the percona repo + +# - import_playbook: test_prep.yml + +- hosts: all + become: true + become_method: sudo + + tasks: + + - name: include tasks for test env setup + include_tasks: ../tasks/test_prep.yml + + - name: install yum-utils for rhel + yum: + name: yum-utils + state: present + when: ansible_os_family == "RedHat" + +# - name: Install MySQL apt repository package +# apt: +# deb: https://repo.mysql.com/mysql-apt-config_0.8.22-1_all.deb +# when: ansible_os_family == "Debian" + + - name: Install MySQL RHEL 6 yum repository package + yum: name=https://repo.mysql.com/mysql80-community-release-el6.rpm state=present + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6" + + - name: Install MySQL RHEL 7 yum repository package + yum: name=https://repo.mysql.com//mysql80-community-release-el7.rpm state=present + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7" + + - name: Import GPG key for RPM 7 (Centos 7) + rpm_key: + state: present + key: https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 + when: ansible_os_family == "RedHat" and ansible_distribution_major_version >= "7" + + - name: Import key for mysql-repo add GPG key (Debian) + apt_key: + url: http://repo.mysql.com/RPM-GPG-KEY-mysql-2023 + state: present + when: ansible_os_family == "Debian" + + - name: Import GPG key for RPM 8-3 + rpm_key: + state: present + key: https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 + when: ansible_os_family == "RedHat" and ansible_distribution_major_version >= "8" + + - name: Install MySQL RHEL 8 yum repository package + yum: name=https://repo.mysql.com/mysql80-community-release-el8.rpm state=present + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "8" + + - name: Install MySQL RHEL 9 yum repository package + yum: name=https://repo.mysql.com/mysql80-community-release-el9.rpm state=present + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "9" + + - name: epel 7 repo + yum: name=https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm state=present + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7" + + - name: setup epel 8 repo + yum: name=https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm state=present + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "8" + + - name: download and extract world database + command: "{{ item }}" + with_items: + - wget --no-check-certificate -P /package-testing https://raw.githubusercontent.com/Percona-QA/percona-qa/master/sample_db/world.sql + + - name: disable the mysql module on RHEL/CentOS 8 + command: /usr/bin/dnf module disable mysql -y + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "8" + + - name: install deb packages + apt: + name: "{{ packages }}" + update_cache: yes + vars: + packages: + - vim + - pv + - unzip + when: ansible_os_family == "Debian" + +# - name: install MySQL Community deb packages +# apt: +# name: "{{ packages }}" +# update_cache: yes +# vars: +# packages: +# - mysql-community-server +# when: ansible_os_family == "Debian" + + - name: Install mysql deb packages using tarballs ubuntu 22.04 + command: wget https://cdn.mysql.com/archives/mysql-8.2/mysql-server_8.2.0-1ubuntu22.04_amd64.deb-bundle.tar + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == '22.04' + + - name: Install mysql deb packages using tarballs ubuntu 23.04 + command: wget https://cdn.mysql.com/archives/mysql-8.2/mysql-server_8.2.0-1ubuntu23.04_amd64.deb-bundle.tar + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == '23.04' + + - name: Install mysql deb packages using tarballs Debian 11 + command: wget https://cdn.mysql.com/archives/mysql-8.2/mysql-server_8.2.0-1debian11_amd64.deb-bundle.tar + when: ansible_distribution == 'Debian' and ansible_distribution_version == '11' + + + - name: Install mysql deb packages using tarballs Debian 12 + command: wget https://cdn.mysql.com/archives/mysql-8.2/mysql-server_8.2.0-1debian12_amd64.deb-bundle.tar + when: ansible_distribution == 'Debian' and ansible_distribution_version == '12.0' + + - name: extract mysql deb packages for ubuntu 22.04 + command: tar -xvf mysql-server_8.2.0-1ubuntu22.04_amd64.deb-bundle.tar -C /tmp + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == '22.04' + + - name: extract mysql deb packages for ubuntu 23.04 + command: tar -xvf mysql-server_8.2.0-1ubuntu23.04_amd64.deb-bundle.tar -C /tmp + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == '23.04' + + - name: extract mysql deb packages for Debian 11 + command: tar -xvf mysql-server_8.2.0-1debian11_amd64.deb-bundle.tar -C /tmp + when: ansible_distribution == 'Debian' and ansible_distribution_version == '11' + + - name: extract mysql deb packages for Debian 12 + command: tar -xvf mysql-server_8.2.0-1debian12_amd64.deb-bundle.tar -C /tmp + when: ansible_distribution == 'Debian' and ansible_distribution_version == '12.0' + + - name: install mysql deb packages + shell: dpkg -i /tmp/*.deb || apt-get install -f -y + environment: + DEBIAN_FRONTEND: noninteractive + when: ansible_os_family == "Debian" + + - name: Enable Innovation LTS repo for RHEL + shell: yum-config-manager --disable mysql80-community && yum-config-manager --enable mysql-innovation-community + when: ansible_os_family == "RedHat" and ansible_distribution_major_version >= "7" + + - name: install packages in centos + yum: + name: "{{ packages }}" + state: present + vars: + packages: + - libselinux-python + - pv + - unzip + - mysql-community-server-8.2.0-1.el7 + when: ansible_os_family == "RedHat" and ansible_distribution_major_version <= "7" + + - name: install packages in rhel/centos 8 + yum: + name: "{{ packages }}" + state: present + vars: + packages: + - python3-libselinux + - mysql-community-server-8.2.0-1.el8 + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "8" + + - name: install packages in rhel/centos 9 + yum: + name: "{{ packages }}" + state: present + vars: + packages: + - python3-libselinux + - mysql-community-server-8.2.0-1.el9 + when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "9" + + - name: Start the MySQL service + service: name=mysql state=started + when: ansible_os_family == "Debian" + + - name: Start the MySQL service + service: name=mysqld state=started + when: ansible_os_family == "RedHat" + + - name: set root password on centos + command: /package-testing/setpass_57.sh + when: ansible_os_family == "RedHat" + + - name: Extract version number using shell commands + shell: cat ../VERSIONS | grep -oP 'PS_INN_LTS_VER="\K(\d+)\.(\d+)' | tr -d '.' + register: major_release_version + + - name: Set major_release_version variable + set_fact: + major_release_version: "{{ major_release_version.stdout }}" + + - name: Extract values using shell command for repo name used for innovation/lts release + shell: grep 'PS_INN_LTS_REPO=' ../VERSIONS | cut -d'=' -f2 | tr -d '"' + register: ps_inn_lts_repo_name + + - name: Set pxb_inn_lts_repo_name variable + set_fact: + ps_inn_lts_repo_name: "{{ ps_inn_lts_repo_name.stdout }}" + + - name: include tasks for enabling PS {{ major_release_version }} main repo + include: ../tasks/enable_ps_innovation_repo_main.yml + when: lookup('env', 'install_repo') == "main" + + - name: include tasks for enabling PS {{ major_release_version }} test repo + include: ../tasks/enable_ps_innovation_repo_testing.yml + when: lookup('env', 'install_repo') == "testing" or lookup('env', 'install_repo') == "" + + - name: include tasks for enabling PS {{ major_release_version }} experimental repo + include: ../tasks/enable_ps_innovation_repo_experimental.yml + when: lookup('env', 'install_repo') == "experimental" + + - name: install Percona XtraBackup {{ major_release_version }} packages + include_tasks: ../tasks/install_pxb_innovation_lts.yml + when: lookup('env', 'install_repo') != "experimental" + + - name: install plugins, import world database + command: /package-testing/plugins_test.sh + + - name: check that Percona XtraBackup version is correct + command: /package-testing/version_check.sh pxb{{ major_release_version }} + + - name: check that Percona XtraBackup package versions are correct + command: /package-testing/package_check.sh pxb{{ major_release_version }} + + - name: run backup + command: /usr/bin/xtrabackup --backup --user=root --target-dir=/tmp/backups/ + + - name: prepare the backup + command: /usr/bin/xtrabackup --prepare --user=root --target-dir=/tmp/backups/ diff --git a/test_binary_tarball.sh b/test_binary_tarball.sh index e37a6088f..785456ca2 100755 --- a/test_binary_tarball.sh +++ b/test_binary_tarball.sh @@ -13,19 +13,23 @@ log="/tmp/binary_check.log" source "${SCRIPT_PWD}"/VERSIONS -if [ "$1" = "pxb81" ]; then - product=pxb81 - version=${PXB81_VER} - major_version="${PXB81_VER}" - minor_version="${PXB81_PKG_VER}" +if [[ "$1" =~ ^pxb8[1-9]{1}$ ]]; then + product=$1 + data="pxb81" + data2=${data:3:2} + product_version_split="${data2:0:1}.${data2:1:1}" + + version=${PXB_INN_LTS_VER} + major_version="${PXB_INN_LTS_MAJ_VER}" + minor_version="${PXB_INN_LTS_PKG_VER}" echo "Downloading ${1} latest version..." >> "${log}" if [ "$2" = "main" ]; then if [ "$3" = "normal" ]; then - wget https://downloads.percona.com/downloads/Percona-XtraBackup-8.1/Percona-XtraBackup-${major_version}-${minor_version}/binary/tarball/percona-xtrabackup-${major_version}-$ {minor_version}-Linux-x86_64.glibc2.17.tar.gz + wget https://downloads.percona.com/downloads/Percona-XtraBackup-${product_version_split}/Percona-XtraBackup-${major_version}-${minor_version}/binary/tarball/percona-xtrabackup-${major_version}-$ {minor_version}-Linux-x86_64.glibc2.17.tar.gz tarball_dir="percona-xtrabackup-${major_version}-${minor_version}-Linux-x86_64.glibc2.17" else # Download minimal version - wget https://downloads.percona.com/downloads/Percona-XtraBackup-8.1/Percona-XtraBackup-${major_version}-${minor_version}/binary/tarball/percona-xtrabackup-${major_version}-$ {minor_version}-Linux-x86_64.glibc2.17-minimal.tar.gz + wget https://downloads.percona.com/downloads/Percona-XtraBackup-${product_version_split}/Percona-XtraBackup-${major_version}-${minor_version}/binary/tarball/percona-xtrabackup-${major_version}-$ {minor_version}-Linux-x86_64.glibc2.17-minimal.tar.gz tarball_dir="percona-xtrabackup-${major_version}-${minor_version}-Linux-x86_64.glibc2.17-minimal" fi else