From 563d9501b9be2d1028382faaf0da9ea6defaa6a2 Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Fri, 12 Jul 2024 21:38:40 +0530 Subject: [PATCH] Fix Downgrade problem from v20 in semi-sync plugin (#16357) Signed-off-by: Manan Gupta --- .../upgrade_downgrade_test_semi_sync.yml | 164 ++++++++++++++++++ config/mycnf/mysql8026.cnf | 6 +- .../reparent/semisync/semi_sync_test.go | 114 ++++++++++++ 3 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/upgrade_downgrade_test_semi_sync.yml create mode 100644 go/test/endtoend/reparent/semisync/semi_sync_test.go diff --git a/.github/workflows/upgrade_downgrade_test_semi_sync.yml b/.github/workflows/upgrade_downgrade_test_semi_sync.yml new file mode 100644 index 00000000000..ffbd9449f61 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_semi_sync.yml @@ -0,0 +1,164 @@ +name: Semi Sync Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Semi Sync Upgrade Downgrade Testing') + cancel-in-progress: true + +permissions: read-all + +jobs: + upgrade_downgrade_test_e2e: + timeout-minutes: 60 + name: Run Semi Sync Upgrade Downgrade Test + runs-on: gh-hosted-runners-16cores-1 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + if: steps.skip-workflow.outputs.skip-workflow == 'false' + id: output-previous-release-ref + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_semi_sync.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v5 + with: + go-version: 1.22.5 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v5 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release enable-only tools + sudo apt-get update + sudo apt-get install -y percona-xtrabackup-80 + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@v4 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Copy last releases vttablet + - name: Copy last release's VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttabletold + vttabletold --version + + - name: Run semi sync tests + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + set -x + source build.env + go test -v -count=1 -run="" ./go/test/endtoend/reparent/semisync -alsologtostderr diff --git a/config/mycnf/mysql8026.cnf b/config/mycnf/mysql8026.cnf index c7755be488f..2488db029b9 100644 --- a/config/mycnf/mysql8026.cnf +++ b/config/mycnf/mysql8026.cnf @@ -20,13 +20,13 @@ default_authentication_plugin = mysql_native_password # # VTTablet will enable semi-sync at the proper time when replication is set up, # or when a primary is promoted or demoted based on the durability policy configured. -plugin-load = rpl_semi_sync_source=semisync_source.so;rpl_semi_sync_replica=semisync_replica.so +plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so # MySQL 8.0.26 and later will not load plugins during --initialize # which makes these options unknown. Prefixing with --loose # tells the server it's fine if they are not understood. -loose_rpl_semi_sync_source_timeout = 1000000000000000000 -loose_rpl_semi_sync_source_wait_no_replica = 1 +loose_rpl_semi_sync_master_timeout = 1000000000000000000 +loose_rpl_semi_sync_master_wait_no_slave = 1 # In order to protect against any errand GTIDs we will start the mysql instance # in super-read-only mode. diff --git a/go/test/endtoend/reparent/semisync/semi_sync_test.go b/go/test/endtoend/reparent/semisync/semi_sync_test.go new file mode 100644 index 00000000000..b0cb9cc1aec --- /dev/null +++ b/go/test/endtoend/reparent/semisync/semi_sync_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package semisync + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/reparent/utils" +) + +func TestSemiSyncUpgradeDowngrade(t *testing.T) { + defer cluster.PanicHandler(t) + clusterInstance := utils.SetupReparentCluster(t, "semi_sync") + defer utils.TeardownCluster(clusterInstance) + tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets + + // Verify that replication is running as intended. + utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) + + replica := tablets[1] + // Verify we are using the correct vttablet version. + verifyVttabletVersion(t, replica, 20) + // Check the plugin loaded in vttablet. + require.EqualValues(t, mysql.SemiSyncTypeMaster, semiSyncExtensionLoaded(t, replica)) + + t.Run("Downgrade to v19", func(t *testing.T) { + // change vttablet binary and downgrade it. + changeVttabletBinary(t, replica, "vttabletold") + // Verify we are using the older vttablet version. + verifyVttabletVersion(t, replica, 19) + // Verify that replication is running as intended. + utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) + // Check the plugin loaded in vttablet. + require.EqualValues(t, mysql.SemiSyncTypeMaster, semiSyncExtensionLoaded(t, replica)) + }) + + t.Run("Upgrade to v19", func(t *testing.T) { + // change vttablet binary and downgrade it. + changeVttabletBinary(t, replica, "vttablet") + // Verify we are using the older vttablet version. + verifyVttabletVersion(t, replica, 20) + // Verify that replication is running as intended. + utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) + // Check the plugin loaded in vttablet. + require.EqualValues(t, mysql.SemiSyncTypeMaster, semiSyncExtensionLoaded(t, replica)) + }) + + t.Run("Change the semi-sync plugin", func(t *testing.T) { + // Change MySQL plugins loaded. + utils.RunSQLs(context.Background(), t, []string{ + `SET GLOBAL READ_ONLY=OFF`, + `STOP REPLICA;`, + `UNINSTALL PLUGIN rpl_semi_sync_master;`, + `UNINSTALL PLUGIN rpl_semi_sync_slave;`, + `INSTALL PLUGIN rpl_semi_sync_source SONAME 'semisync_source.so';`, + `INSTALL PLUGIN rpl_semi_sync_replica SONAME 'semisync_replica.so';`, + `START REPLICA;`, + }, replica) + // Check the plugin loaded in vttablet. + require.EqualValues(t, mysql.SemiSyncTypeSource, semiSyncExtensionLoaded(t, replica)) + // Verify that replication is running as intended. + utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) + }) +} + +// semiSyncExtensionLoaded checks if the semisync extension has been loaded. +// It should work for both MariaDB and MySQL. +func semiSyncExtensionLoaded(t *testing.T, replica *cluster.Vttablet) mysql.SemiSyncType { + qr := utils.RunSQL(context.Background(), t, `SHOW VARIABLES LIKE 'rpl_semi_sync_%_enabled'`, replica) + for _, row := range qr.Rows { + if row[0].ToString() == "rpl_semi_sync_source_enabled" { + return mysql.SemiSyncTypeSource + } + if row[0].ToString() == "rpl_semi_sync_master_enabled" { + return mysql.SemiSyncTypeMaster + } + } + return mysql.SemiSyncTypeOff +} + +func changeVttabletBinary(t *testing.T, replica *cluster.Vttablet, binary string) { + t.Helper() + err := replica.VttabletProcess.TearDown() + require.NoError(t, err) + replica.VttabletProcess.Binary = binary + err = replica.VttabletProcess.Setup() + require.NoError(t, err) +} + +func verifyVttabletVersion(t *testing.T, replica *cluster.Vttablet, version int) { + t.Helper() + verGot, err := cluster.GetMajorVersion(replica.VttabletProcess.Binary) + require.NoError(t, err) + require.EqualValues(t, version, verGot) +}