[release-16.0] Backup flaky test (#14819)

Signed-off-by: Manan Gupta <[email protected]>
Co-authored-by: Rameez Sajwani <[email protected]>

go/test/endtoend/cluster/cluster_util.go

@@ -43,7 +43,7 @@ var (
 	dbCredentialFile         string
 	InsertTabletTemplateKsID = `insert into %s (id, msg) values (%d, '%s') /* id:%d */`
 	defaultOperationTimeout  = 60 * time.Second
-	defeaultRetryDelay       = 1 * time.Second
+	defaultRetryDelay        = 1 * time.Second
 )
 
 // Restart restarts vttablet and mysql.
@@ -54,15 +54,17 @@ func (tablet *Vttablet) Restart() error {
 
 	if tablet.MysqlctlProcess.TabletUID > 0 {
 		tablet.MysqlctlProcess.Stop()
+		tablet.MysqlctldProcess.WaitForMysqlCtldShutdown()
 		tablet.VttabletProcess.TearDown()
-		os.RemoveAll(tablet.VttabletProcess.Directory)
+		tablet.MysqlctldProcess.CleanupFiles(tablet.TabletUID)
 
 		return tablet.MysqlctlProcess.Start()
 	}
 
 	tablet.MysqlctldProcess.Stop()
+	tablet.MysqlctldProcess.WaitForMysqlCtldShutdown()
 	tablet.VttabletProcess.TearDown()
-	os.RemoveAll(tablet.VttabletProcess.Directory)
+	tablet.MysqlctldProcess.CleanupFiles(tablet.TabletUID)
 
 	return tablet.MysqlctldProcess.Start()
 }
@@ -436,6 +438,6 @@ func WaitForHealthyShard(vtctldclient *VtctldClientProcess, keyspace, shard stri
 		default:
 		}
 
-		time.Sleep(defeaultRetryDelay)
+		time.Sleep(defaultRetryDelay)
 	}
 }

go/test/endtoend/cluster/mysqlctl_process.go

@@ -217,11 +217,7 @@ func (mysqlctl *MysqlctlProcess) BinaryLogsPath() string {
 
 // CleanupFiles clean the mysql files to make sure we can start the same process again
 func (mysqlctl *MysqlctlProcess) CleanupFiles(tabletUID int) {
-	os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/data", tabletUID)))
-	os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/relay-logs", tabletUID)))
-	os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/tmp", tabletUID)))
-	os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/bin-logs", tabletUID)))
-	os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/innodb", tabletUID)))
+	os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tabletUID)))
 }
 
 // Connect returns a new connection to the underlying MySQL server

go/test/endtoend/cluster/mysqlctld_process.go

@@ -172,3 +172,25 @@ func (mysqlctld *MysqlctldProcess) IsHealthy() bool {
 	_, err := mysql.Connect(context.Background(), &params)
 	return err == nil
 }
+
+// HasShutdown checks if the process has been set to nil
+func (mysqlctld *MysqlctldProcess) hasShutdown() bool {
+	return mysqlctld.process == nil
+}
+
+// WaitForMysqlCtldShutdown waits for mysqlctld to have shutdown.
+func (mysqlctld *MysqlctldProcess) WaitForMysqlCtldShutdown() bool {
+	tmr := time.NewTimer(defaultOperationTimeout)
+	defer tmr.Stop()
+	for {
+		if mysqlctld.hasShutdown() {
+			return true
+		}
+		select {
+		case <-tmr.C:
+			return false
+		default:
+		}
+		time.Sleep(defaultRetryDelay)
+	}
+}

go/test/endtoend/mysqlctld/mysqlctld_test.go

@@ -133,6 +133,7 @@ func TestRestart(t *testing.T) {
 	defer cluster.PanicHandler(t)
 	err := primaryTablet.MysqlctldProcess.Stop()
 	require.Nil(t, err)
+	require.Truef(t, primaryTablet.MysqlctldProcess.WaitForMysqlCtldShutdown(), "Mysqlctld has not stopped...")
 	primaryTablet.MysqlctldProcess.CleanupFiles(primaryTablet.TabletUID)
 	err = primaryTablet.MysqlctldProcess.Start()
 	require.Nil(t, err)