From 4dcc3ff22ec393c8da8b1246af1cdd2343888ffd Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 19 Nov 2023 17:08:29 +0200 Subject: [PATCH 1/2] snapshot connection: revert to explciit table locks if 'FLUSH TABLES WITH READ LOCK' is rejected Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/vstreamer/snapshot_conn.go | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go b/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go index b9a3a70ea98..31881e80464 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go +++ b/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go @@ -19,12 +19,15 @@ package vstreamer import ( "context" "fmt" + "math" + "strings" "sync/atomic" "time" "github.com/spf13/pflag" "vitess.io/vitess/go/mysql/replication" + "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/dbconfigs" @@ -241,8 +244,40 @@ func (conn *snapshotConn) startSnapshotAllTables(ctx context.Context) (gtid stri log.Infof("Locking all tables") if _, err := lockConn.ExecuteFetch("FLUSH TABLES WITH READ LOCK", 1, false); err != nil { + attemptExplicitTablesLocks := false + if sqlErr, ok := err.(*sqlerror.SQLError); ok && sqlErr.Number() == sqlerror.ERAccessDeniedError { + // Access denied. On some systems this is either because the user doesn't have SUPER or RELOAD privileges. + // On some other systems, namely RDS, the command is just unsupported. + // There is an alternative way: run a `LOCK TABLES tbl1 READ, tbl2 READ, ...` for all tables. It not as + // efficient, and make a huge query, but still better than nothing. + attemptExplicitTablesLocks = true + } log.Infof("Error locking all tables") - return "", err + if !attemptExplicitTablesLocks { + return "", err + } + // get list of all tables + rs, err := conn.ExecuteFetch("show full tables", math.MaxInt32, true) + if err != nil { + return "", err + } + + var lockClauses []string + for _, row := range rs.Rows { + tableName := row[0].ToString() + tableType := row[1].ToString() + if tableType != "BASE TABLE" { + continue + } + tableName = sqlparser.String(sqlparser.NewIdentifierCS(tableName)) + lockClause := fmt.Sprintf("%s read", tableName) + lockClauses = append(lockClauses, lockClause) + } + query := fmt.Sprintf("lock tables %s", strings.Join(lockClauses, ",")) + if _, err := lockConn.ExecuteFetch(query, 1, false); err != nil { + log.Infof("Error explicitly locking all %v tables", len(lockClauses)) + return "", err + } } mpos, err := lockConn.PrimaryPosition() if err != nil { From deaaa4ad3d91096fb4af568709dd3406e297a1fc Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 22 Nov 2023 13:10:15 +0200 Subject: [PATCH 2/2] log error; consider empty list Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/vstreamer/snapshot_conn.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go b/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go index 31881e80464..5c9885ee82e 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go +++ b/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go @@ -34,6 +34,7 @@ import ( "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vterrors" ) // If the current binary log is greater than this byte size, we @@ -273,10 +274,14 @@ func (conn *snapshotConn) startSnapshotAllTables(ctx context.Context) (gtid stri lockClause := fmt.Sprintf("%s read", tableName) lockClauses = append(lockClauses, lockClause) } - query := fmt.Sprintf("lock tables %s", strings.Join(lockClauses, ",")) - if _, err := lockConn.ExecuteFetch(query, 1, false); err != nil { - log.Infof("Error explicitly locking all %v tables", len(lockClauses)) - return "", err + if len(lockClauses) > 0 { + query := fmt.Sprintf("lock tables %s", strings.Join(lockClauses, ",")) + if _, err := lockConn.ExecuteFetch(query, 1, false); err != nil { + log.Error(vterrors.Wrapf(err, "explicitly locking all %v tables", len(lockClauses))) + return "", err + } + } else { + log.Infof("explicit lock tables: no tables found") } } mpos, err := lockConn.PrimaryPosition()