From 9c2d6cf4e31ed00aa4705770ebe1870c2fefb67b Mon Sep 17 00:00:00 2001
From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com>
Date: Tue, 12 Dec 2023 11:45:19 -0600
Subject: [PATCH] [release-18.0] Add a retry to remove the vttablet directory
 during upgrade/downgrade backup tests (#14753) (#14758)

Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com>
---
 .../upgrade_downgrade_test_backups_manual.yml |  1 +
 ...grade_test_backups_manual_next_release.yml |  1 +
 examples/backups/stop_tablets.sh              | 20 ++++++++++++++++++-
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml
index 039e416cb1c..82d243066e4 100644
--- a/.github/workflows/upgrade_downgrade_test_backups_manual.yml
+++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml
@@ -82,6 +82,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/upgrade_downgrade_test_backups_manual.yml'
+            - 'examples/**'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml
index e4a9bd11fb3..f00b53c9313 100644
--- a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml
+++ b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml
@@ -85,6 +85,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml'
+            - 'examples/**'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
diff --git a/examples/backups/stop_tablets.sh b/examples/backups/stop_tablets.sh
index 6a3ced6ab74..d387128309c 100755
--- a/examples/backups/stop_tablets.sh
+++ b/examples/backups/stop_tablets.sh
@@ -30,7 +30,25 @@ for tablet in 100 200 300; do
       CELL=zone1 TABLET_UID=$uid ../common/scripts/mysqlctl-down.sh
       echo "Removing tablet directory zone1-$uid"
       vtctldclient DeleteTablets --allow-primary zone1-$uid
-      rm -Rf $VTDATAROOT/vt_0000000$uid
+
+      for ((i=0; i<30; i++)); do
+          # Redirect stderr to a temporary file
+          temp_file=$(mktemp)
+          rm -Rf $VTDATAROOT/vt_0000000$uid 2>"$temp_file"
+
+          if grep -q 'Directory not empty' "$temp_file"; then
+              echo "Directory not empty, retrying..."
+          elif [ ! -s "$temp_file" ]; then
+              echo "Deletion succeeded."
+              rm -f "$temp_file"
+              break
+          else
+              echo "An error occurred."
+              cat "$temp_file"
+          fi
+          rm -f "$temp_file"
+          sleep 1
+      done
     done
   fi
 done