From e6392f3ddc0435feb9e31c0073e93b723928ead9 Mon Sep 17 00:00:00 2001
From: Florent Poinsard <35779988+frouioui@users.noreply.github.com>
Date: Tue, 12 Dec 2023 00:28:28 -0600
Subject: [PATCH] Add a retry to remove the vttablet directory during
 upgrade/downgrade backup tests (#14753)

Signed-off-by: Florent Poinsard <florent.poinsard@outlook.fr>
---
 .../upgrade_downgrade_test_backups_manual.yml |  1 +
 ...grade_test_backups_manual_next_release.yml |  1 +
 examples/backups/stop_tablets.sh              | 20 ++++++++++++++++++-
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml
index 6aaa9309cb5..31f23533621 100644
--- a/.github/workflows/upgrade_downgrade_test_backups_manual.yml
+++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml
@@ -80,6 +80,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/upgrade_downgrade_test_backups_manual.yml'
+            - 'examples/**'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml
index 47a44f09acc..0890448a427 100644
--- a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml
+++ b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml
@@ -83,6 +83,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml'
+            - 'examples/**'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
diff --git a/examples/backups/stop_tablets.sh b/examples/backups/stop_tablets.sh
index 2a45e9e68d2..9c90bcb5b33 100755
--- a/examples/backups/stop_tablets.sh
+++ b/examples/backups/stop_tablets.sh
@@ -30,7 +30,25 @@ for tablet in 100 200 300; do
       CELL=zone1 TABLET_UID=$uid ../common/scripts/mysqlctl-down.sh
       echo "Removing tablet directory zone1-$uid"
       vtctlclient DeleteTablet -- --allow_primary=true zone1-$uid
-      rm -Rf $VTDATAROOT/vt_0000000$uid
+
+      for ((i=0; i<30; i++)); do
+          # Redirect stderr to a temporary file
+          temp_file=$(mktemp)
+          rm -Rf $VTDATAROOT/vt_0000000$uid 2>"$temp_file"
+
+          if grep -q 'Directory not empty' "$temp_file"; then
+              echo "Directory not empty, retrying..."
+          elif [ ! -s "$temp_file" ]; then
+              echo "Deletion succeeded."
+              rm -f "$temp_file"
+              break
+          else
+              echo "An error occurred."
+              cat "$temp_file"
+          fi
+          rm -f "$temp_file"
+          sleep 1
+      done
     done
   fi
 done