From ab6eb5d96e537ac6b52c820d69dd1f1cedec2dde Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 26 Nov 2023 16:12:10 +0200 Subject: [PATCH 1/7] Introducing new name format for internal vitess tables. At this time (v19), the new format is accepted and recognized, but never generated. Names will be generated with the new format starting v20 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/schema/name.go | 30 ++++++++++++ go/vt/schema/name_test.go | 79 ++++++++++++++++++++++++++++++ go/vt/schema/online_ddl.go | 16 ++++++ go/vt/schema/online_ddl_test.go | 69 ++++++++++++++++---------- go/vt/schema/tablegc.go | 27 ++++++++-- go/vt/schema/tablegc_test.go | 87 ++++++++++++++++++++++++++------- 6 files changed, 262 insertions(+), 46 deletions(-) diff --git a/go/vt/schema/name.go b/go/vt/schema/name.go index 42d3878b302..4e8ec14db14 100644 --- a/go/vt/schema/name.go +++ b/go/vt/schema/name.go @@ -17,6 +17,7 @@ limitations under the License. package schema import ( + "regexp" "strings" "time" @@ -27,6 +28,14 @@ const ( readableTimeFormat = "20060102150405" ) +const ( + InternalTableNameExpression string = `^_vt_([a-zA-Z0-9]{3})_([0-f]{32})_([0-9]{14})_$` +) + +var ( + internalTableNameRegexp = regexp.MustCompile(InternalTableNameExpression) +) + // CreateUUIDWithDelimiter creates a globally unique ID, with a given delimiter // example results: // - 1876a01a-354d-11eb-9a79-f8e4e33000bb (delimiter = "-") @@ -61,6 +70,9 @@ func ToReadableTimestamp(t time.Time) string { // - Table GC (renamed before drop) // Apps such as VStreamer may choose to ignore such tables. func IsInternalOperationTableName(tableName string) bool { + if internalTableNameRegexp.MatchString(tableName) { + return true + } if IsGCTableName(tableName) { return true } @@ -69,3 +81,21 @@ func IsInternalOperationTableName(tableName string) bool { } return false } + +// AnalyzeInternalTableName analyzes a table name, and assumign it's a vitess internal table name, extracts +// the hint, uuid and time out of the name. +// An internal table name can be e.g. `_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_`, analyzed like so: +// - hint is `hld` +// - UUID is `6ace8bcef73211ea87e9f875a4d24e90` +// - Time is 2020-09-15 12:04:10 +func AnalyzeInternalTableName(tableName string) (isInternalTable bool, hint string, uuid string, t time.Time, err error) { + submatch := internalTableNameRegexp.FindStringSubmatch(tableName) + if len(submatch) == 0 { + return false, hint, uuid, t, nil + } + t, err = time.Parse(readableTimeFormat, submatch[3]) + if err != nil { + return false, hint, uuid, t, err + } + return true, submatch[1], submatch[2], t, nil +} diff --git a/go/vt/schema/name_test.go b/go/vt/schema/name_test.go index ab72f80644e..24571b91b9f 100644 --- a/go/vt/schema/name_test.go +++ b/go/vt/schema/name_test.go @@ -18,6 +18,7 @@ package schema import ( "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -69,6 +70,14 @@ func TestIsInternalOperationTableName(t *testing.T) { "_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", "_vt_EVAC_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", "_vt_PURGE_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + "_vt_drp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_evc_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_vrp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_gho_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_ghc_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_xyz_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", } for _, tableName := range names { assert.True(t, IsInternalOperationTableName(tableName)) @@ -93,3 +102,73 @@ func TestIsInternalOperationTableName(t *testing.T) { assert.False(t, IsInternalOperationTableName(tableName)) } } + +func TestAnalyzeInternalTableName(t *testing.T) { + baseTime, err := time.Parse(time.RFC1123, "Tue, 15 Sep 2020 12:04:10 UTC") + assert.NoError(t, err) + tt := []struct { + tableName string + hint string + t time.Time + isInternal bool + }{ + { + tableName: "_84371a37_6153_11eb_9917_f875a4d24e90_20210128122816_vrepl", + isInternal: false, + }, + { + tableName: "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + isInternal: false, + }, + { + tableName: "_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + isInternal: false, + }, + { + tableName: "_vt_EVAC_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + isInternal: false, + }, + { + tableName: "_vt_PURGE_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + isInternal: false, + }, + { + tableName: "_vt_drop_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + isInternal: false, + }, + { + tableName: "_vt_drp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + hint: "drp", + t: baseTime, + isInternal: true, + }, + { + tableName: "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + hint: "hld", + t: baseTime, + isInternal: true, + }, + { + tableName: "_vt_xyz_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + hint: "xyz", + t: baseTime, + isInternal: true, + }, + { + tableName: "_vt_xyz_6ace8bcef73211ea87e9f875a4d24e90_20200915129999_", + isInternal: false, + }, + } + for _, ts := range tt { + t.Run(ts.tableName, func(t *testing.T) { + isInternal, hint, uuid, tm, err := AnalyzeInternalTableName(ts.tableName) + assert.Equal(t, ts.isInternal, isInternal) + if ts.isInternal { + assert.NoError(t, err) + assert.True(t, IsGCUUID(uuid)) + assert.Equal(t, ts.hint, hint) + assert.Equal(t, ts.t, tm) + } + }) + } +} diff --git a/go/vt/schema/online_ddl.go b/go/vt/schema/online_ddl.go index a06866e996a..66d5bd2982c 100644 --- a/go/vt/schema/online_ddl.go +++ b/go/vt/schema/online_ddl.go @@ -37,6 +37,16 @@ var ( migrationContextValidatorRegexp = regexp.MustCompile(`^[\w:-]*$`) ) +var ( + onlineDDLInternalTableHintsMap = map[string]bool{ + "vrp": true, // vreplication + "gho": true, // gh-ost + "ghc": true, // gh-ost + "del": true, // gh-ost + "new": true, // pt-osc + } +) + var ( // ErrDirectDDLDisabled is returned when direct DDL is disabled, and a user attempts to run a DDL statement ErrDirectDDLDisabled = errors.New("direct DDL is disabled") @@ -461,6 +471,12 @@ func OnlineDDLToGCUUID(uuid string) string { // by pt-online-schema-change. // There is no guarantee that the tables _was indeed_ generated by an online DDL flow. func IsOnlineDDLTableName(tableName string) bool { + // Try new naming format (e.g. `_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_`): + // The new naming format is accepted in v19, and actually _used_ in v20 + if isInternal, hint, _, _, _ := AnalyzeInternalTableName(tableName); isInternal { + return isInternal && onlineDDLInternalTableHintsMap[hint] + } + if onlineDDLGeneratedTableNameRegexp.MatchString(tableName) { return true } diff --git a/go/vt/schema/online_ddl_test.go b/go/vt/schema/online_ddl_test.go index c616d64a698..6b15f865b26 100644 --- a/go/vt/schema/online_ddl_test.go +++ b/go/vt/schema/online_ddl_test.go @@ -101,31 +101,50 @@ func TestGetActionStr(t *testing.T) { } func TestIsOnlineDDLTableName(t *testing.T) { - names := []string{ - "_4e5dcf80_354b_11eb_82cd_f875a4d24e90_20201203114014_gho", - "_4e5dcf80_354b_11eb_82cd_f875a4d24e90_20201203114014_ghc", - "_4e5dcf80_354b_11eb_82cd_f875a4d24e90_20201203114014_del", - "_4e5dcf80_354b_11eb_82cd_f875a4d24e90_20201203114013_new", - "_84371a37_6153_11eb_9917_f875a4d24e90_20210128122816_vrepl", - "_table_old", - "__table_old", - } - for _, tableName := range names { - assert.True(t, IsOnlineDDLTableName(tableName)) - } - irrelevantNames := []string{ - "t", - "_table_new", - "__table_new", - "_table_gho", - "_table_ghc", - "_table_del", - "_table_vrepl", - "table_old", - } - for _, tableName := range irrelevantNames { - assert.False(t, IsOnlineDDLTableName(tableName)) - } + t.Run("accept", func(t *testing.T) { + names := []string{ + "_vt_vrp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_gho_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_ghc_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_del_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_new_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_4e5dcf80_354b_11eb_82cd_f875a4d24e90_20201203114014_gho", + "_4e5dcf80_354b_11eb_82cd_f875a4d24e90_20201203114014_ghc", + "_4e5dcf80_354b_11eb_82cd_f875a4d24e90_20201203114014_del", + "_4e5dcf80_354b_11eb_82cd_f875a4d24e90_20201203114013_new", + "_84371a37_6153_11eb_9917_f875a4d24e90_20210128122816_vrepl", + "_table_old", + "__table_old", + } + for _, tableName := range names { + t.Run(tableName, func(t *testing.T) { + assert.True(t, IsOnlineDDLTableName(tableName)) + }) + } + }) + t.Run("reject", func(t *testing.T) { + irrelevantNames := []string{ + "_vt_vrp_6ace8bcef73211ea87e9f875a4d24e90_20200915999999_", // time error + "_vt_xyz_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", // unrecognized hint + "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", // GC table + "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", // GC table + "_vt_evc_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", // GC table + "_vt_drp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", // GC table + "t", + "_table_new", + "__table_new", + "_table_gho", + "_table_ghc", + "_table_del", + "_table_vrepl", + "table_old", + } + for _, tableName := range irrelevantNames { + t.Run(tableName, func(t *testing.T) { + assert.False(t, IsOnlineDDLTableName(tableName)) + }) + } + }) } func TestGetRevertUUID(t *testing.T) { diff --git a/go/vt/schema/tablegc.go b/go/vt/schema/tablegc.go index 872fb42dbe5..c2f9065cb6a 100644 --- a/go/vt/schema/tablegc.go +++ b/go/vt/schema/tablegc.go @@ -45,7 +45,8 @@ const ( ) const ( - GCTableNameExpression string = `^_vt_(HOLD|PURGE|EVAC|DROP)_([0-f]{32})_([0-9]{14})$` + GCTableNameExpression string = `^_vt_(HOLD|PURGE|EVAC|DROP)_([0-f]{32})_([0-9]{14})$` + NewGCTableNameExpression string = `^_vt_(hld|prg|evc|drp)_([0-f]{32})_([0-9]{14})_$` // =========== use this in conjunction with GCTableNameExpression in vreplicator. Add testing! ) var ( @@ -54,9 +55,13 @@ var ( gcStates = map[string]TableGCState{ string(HoldTableGCState): HoldTableGCState, + "hld": HoldTableGCState, string(PurgeTableGCState): PurgeTableGCState, + "prg": PurgeTableGCState, string(EvacTableGCState): EvacTableGCState, + "evc": EvacTableGCState, string(DropTableGCState): DropTableGCState, + "drp": DropTableGCState, } ) @@ -90,17 +95,33 @@ func GenerateGCTableName(state TableGCState, t time.Time) (tableName string, err // AnalyzeGCTableName analyzes a given table name to see if it's a GC table, and if so, parse out // its state, uuid, and timestamp func AnalyzeGCTableName(tableName string) (isGCTable bool, state TableGCState, uuid string, t time.Time, err error) { + // Try new naming format (e.g. `_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_`): + // The new naming format is accepted in v19, and actually _used_ in v20 + if isInternal, hint, uuid, t, err := AnalyzeInternalTableName(tableName); isInternal { + gcState, ok := gcStates[hint] + return ok, gcState, uuid, t, err + } + // Try old naming formats. These names will not be generated in v20. + // TODO(shlomi): the code below should be remvoed in v21 submatch := gcTableNameRegexp.FindStringSubmatch(tableName) if len(submatch) == 0 { return false, state, uuid, t, nil } + gcState, ok := gcStates[submatch[1]] + if !ok { + return false, state, uuid, t, nil + } t, err = time.Parse(readableTimeFormat, submatch[3]) - return true, TableGCState(submatch[1]), submatch[2], t, err + if err != nil { + return false, state, uuid, t, err + } + return true, gcState, submatch[2], t, nil } // IsGCTableName answers 'true' when the given table name stands for a GC table func IsGCTableName(tableName string) bool { - return gcTableNameRegexp.MatchString(tableName) + isGC, _, _, _, _ := AnalyzeGCTableName(tableName) + return isGC } // GenerateRenameStatementWithUUID generates a "RENAME TABLE" statement, where a table is renamed to a GC table, with preset UUID diff --git a/go/vt/schema/tablegc_test.go b/go/vt/schema/tablegc_test.go index 90b31ff90fa..fb9b2206d80 100644 --- a/go/vt/schema/tablegc_test.go +++ b/go/vt/schema/tablegc_test.go @@ -34,18 +34,36 @@ func TestIsGCTableName(t *testing.T) { assert.True(t, IsGCTableName(tableName)) } } - names := []string{ - "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_202009151204100", - "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915120410 ", - "__vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", - "_vt_DROP_6ace8bcef73211ea87e9f875a4d2_20200915120410", - "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915", - "_vt_OTHER_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", - "_vt_OTHER_6ace8bcef73211ea87e9f875a4d24e90_zz20200915120410", - } - for _, tableName := range names { - assert.False(t, IsGCTableName(tableName)) - } + t.Run("accept", func(t *testing.T) { + names := []string{ + "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + "_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + "_vt_drp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + } + for _, tableName := range names { + t.Run(tableName, func(t *testing.T) { + assert.True(t, IsGCTableName(tableName)) + }) + } + }) + t.Run("reject", func(t *testing.T) { + names := []string{ + "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_202009151204100", + "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915120410 ", + "__vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + "_vt_DROP_6ace8bcef73211ea87e9f875a4d2_20200915120410", + "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915", + "_vt_OTHER_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + "_vt_OTHER_6ace8bcef73211ea87e9f875a4d24e90_zz20200915120410", + "_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_20200915999999", + "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915999999_", + } + for _, tableName := range names { + t.Run(tableName, func(t *testing.T) { + assert.False(t, IsGCTableName(tableName)) + }) + } + }) } func TestAnalyzeGCTableName(t *testing.T) { @@ -55,35 +73,68 @@ func TestAnalyzeGCTableName(t *testing.T) { tableName string state TableGCState t time.Time + isGC bool }{ { tableName: "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", state: DropTableGCState, t: baseTime, + isGC: true, }, { tableName: "_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", state: HoldTableGCState, t: baseTime, + isGC: true, }, { tableName: "_vt_EVAC_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", state: EvacTableGCState, t: baseTime, + isGC: true, }, { tableName: "_vt_PURGE_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", state: PurgeTableGCState, t: baseTime, + isGC: true, + }, + { + tableName: "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915999999", // time error + isGC: false, + }, + { + tableName: "_vt_drp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + state: DropTableGCState, + t: baseTime, + isGC: true, + }, + { + tableName: "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + state: HoldTableGCState, + t: baseTime, + isGC: true, + }, + { + tableName: "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915999999_", // time error + isGC: false, + }, + { + tableName: "_vt_xyz_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + isGC: false, }, } for _, ts := range tt { - isGC, state, uuid, tm, err := AnalyzeGCTableName(ts.tableName) - assert.NoError(t, err) - assert.True(t, isGC) - assert.True(t, IsGCUUID(uuid)) - assert.Equal(t, ts.state, state) - assert.Equal(t, ts.t, tm) + t.Run(ts.tableName, func(t *testing.T) { + isGC, state, uuid, tm, err := AnalyzeGCTableName(ts.tableName) + assert.Equal(t, ts.isGC, isGC) + if ts.isGC { + assert.NoError(t, err) + assert.True(t, IsGCUUID(uuid)) + assert.Equal(t, ts.state, state) + assert.Equal(t, ts.t, tm) + } + }) } } From 1a06719ff438bb6ea7001ada1589ee742f0b0fa6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 27 Nov 2023 08:22:19 +0200 Subject: [PATCH 2/7] buildColInfoMap: exclude new GC table format Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/schema/tablegc_test.go | 35 +++++++++++++++++++ .../tabletmanager/vreplication/vreplicator.go | 8 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/go/vt/schema/tablegc_test.go b/go/vt/schema/tablegc_test.go index fb9b2206d80..f6db4353628 100644 --- a/go/vt/schema/tablegc_test.go +++ b/go/vt/schema/tablegc_test.go @@ -17,6 +17,7 @@ limitations under the License. package schema import ( + "regexp" "testing" "time" @@ -64,6 +65,40 @@ func TestIsGCTableName(t *testing.T) { }) } }) + + t.Run("explicit regexp", func(t *testing.T) { + // NewGCTableNameExpression regexp is used externally by vreplication. Its a redundant form of + // InternalTableNameExpression, but is nonetheless required. We verify it works correctly + re := regexp.MustCompile(NewGCTableNameExpression) + t.Run("accept", func(t *testing.T) { + names := []string{ + "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_evc_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_drp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + } + for _, tableName := range names { + t.Run(tableName, func(t *testing.T) { + assert.True(t, IsGCTableName(tableName)) + assert.True(t, re.MatchString(tableName)) + }) + } + }) + t.Run("reject", func(t *testing.T) { + names := []string{ + "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + "_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + "_vt_vrp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_gho_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + } + for _, tableName := range names { + t.Run(tableName, func(t *testing.T) { + assert.False(t, re.MatchString(tableName)) + }) + } + }) + }) + } func TestAnalyzeGCTableName(t *testing.T) { diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index c4df7e618d5..ba1e4070931 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -329,7 +329,13 @@ type ColumnInfo struct { } func (vr *vreplicator) buildColInfoMap(ctx context.Context) (map[string][]*ColumnInfo, error) { - req := &tabletmanagerdatapb.GetSchemaRequest{Tables: []string{"/.*/"}, ExcludeTables: []string{"/" + schema.GCTableNameExpression + "/"}} + req := &tabletmanagerdatapb.GetSchemaRequest{ + Tables: []string{"/.*/"}, + ExcludeTables: []string{ + "/" + schema.GCTableNameExpression + "/", + "/" + schema.NewGCTableNameExpression + "/", + }, + } schema, err := vr.mysqld.GetSchema(ctx, vr.dbClient.DBName(), req) if err != nil { return nil, err From b55ca8c392e30d5d30b34cc3819a1c5d29a8be45 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 27 Nov 2023 13:59:57 +0200 Subject: [PATCH 3/7] generate RENAME statement for new name format Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/schema/tablegc.go | 46 +++++++++++++++++++++++++++++++++++- go/vt/schema/tablegc_test.go | 8 +++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/go/vt/schema/tablegc.go b/go/vt/schema/tablegc.go index c2f9065cb6a..81aaee15957 100644 --- a/go/vt/schema/tablegc.go +++ b/go/vt/schema/tablegc.go @@ -46,7 +46,7 @@ const ( const ( GCTableNameExpression string = `^_vt_(HOLD|PURGE|EVAC|DROP)_([0-f]{32})_([0-9]{14})$` - NewGCTableNameExpression string = `^_vt_(hld|prg|evc|drp)_([0-f]{32})_([0-9]{14})_$` // =========== use this in conjunction with GCTableNameExpression in vreplicator. Add testing! + NewGCTableNameExpression string = `^_vt_(hld|prg|evc|drp)_([0-f]{32})_([0-9]{14})_$` ) var ( @@ -87,11 +87,41 @@ func generateGCTableName(state TableGCState, uuid string, t time.Time) (tableNam return fmt.Sprintf("_vt_%s_%s_%s", state, uuid, timestamp), nil } +// generateGCTableNameNewFormat creates a GC table name, based on desired state and time, and with optional preset UUID. +// If uuid is given, then it must be in GC-UUID format. If empty, the function auto-generates a UUID. +func generateGCTableNameNewFormat(state TableGCState, uuid string, t time.Time) (tableName string, err error) { + if uuid == "" { + uuid, err = CreateUUIDWithDelimiter("") + } + if err != nil { + return "", err + } + if !IsGCUUID(uuid) { + return "", fmt.Errorf("Not a valid GC UUID format: %s", uuid) + } + timestamp := ToReadableTimestamp(t) + var hint string + for k, v := range gcStates { + if v != state { + continue + } + if len(k) == 3 && k != string(state) { // the "new" format + hint = k + } + } + return fmt.Sprintf("_vt_%s_%s_%s_", hint, uuid, timestamp), nil +} + // GenerateGCTableName creates a GC table name, based on desired state and time, and with random UUID func GenerateGCTableName(state TableGCState, t time.Time) (tableName string, err error) { return generateGCTableName(state, "", t) } +// GenerateGCTableNameNewFormat creates a GC table name, based on desired state and time, and with random UUID +func GenerateGCTableNameNewFormat(state TableGCState, t time.Time) (tableName string, err error) { + return generateGCTableNameNewFormat(state, "", t) +} + // AnalyzeGCTableName analyzes a given table name to see if it's a GC table, and if so, parse out // its state, uuid, and timestamp func AnalyzeGCTableName(tableName string) (isGCTable bool, state TableGCState, uuid string, t time.Time, err error) { @@ -133,11 +163,25 @@ func GenerateRenameStatementWithUUID(fromTableName string, state TableGCState, u return fmt.Sprintf("RENAME TABLE `%s` TO %s", fromTableName, toTableName), toTableName, nil } +// GenerateRenameStatementWithUUIDNewFormat generates a "RENAME TABLE" statement, where a table is renamed to a GC table, with preset UUID +func GenerateRenameStatementWithUUIDNewFormat(fromTableName string, state TableGCState, uuid string, t time.Time) (statement string, toTableName string, err error) { + toTableName, err = generateGCTableNameNewFormat(state, uuid, t) + if err != nil { + return "", "", err + } + return fmt.Sprintf("RENAME TABLE `%s` TO %s", fromTableName, toTableName), toTableName, nil +} + // GenerateRenameStatement generates a "RENAME TABLE" statement, where a table is renamed to a GC table. func GenerateRenameStatement(fromTableName string, state TableGCState, t time.Time) (statement string, toTableName string, err error) { return GenerateRenameStatementWithUUID(fromTableName, state, "", t) } +// GenerateRenameStatement generates a "RENAME TABLE" statement, where a table is renamed to a GC table. +func GenerateRenameStatementNewFormat(fromTableName string, state TableGCState, t time.Time) (statement string, toTableName string, err error) { + return GenerateRenameStatementWithUUIDNewFormat(fromTableName, state, "", t) +} + // ParseGCLifecycle parses a comma separated list of gc states and returns a map of indicated states func ParseGCLifecycle(gcLifecycle string) (states map[TableGCState]bool, err error) { states = make(map[TableGCState]bool) diff --git a/go/vt/schema/tablegc_test.go b/go/vt/schema/tablegc_test.go index f6db4353628..fb9795d4978 100644 --- a/go/vt/schema/tablegc_test.go +++ b/go/vt/schema/tablegc_test.go @@ -33,6 +33,14 @@ func TestIsGCTableName(t *testing.T) { tableName, err := generateGCTableName(state, "", tm) assert.NoError(t, err) assert.True(t, IsGCTableName(tableName)) + + tableName, err = generateGCTableNameNewFormat(state, "6ace8bcef73211ea87e9f875a4d24e90", tm) + assert.NoError(t, err) + assert.Truef(t, IsGCTableName(tableName), "table name: %s", tableName) + + tableName, err = GenerateGCTableNameNewFormat(state, tm) + assert.NoError(t, err) + assert.Truef(t, IsGCTableName(tableName), "table name: %s", tableName) } } t.Run("accept", func(t *testing.T) { From 068433af3c37db523286da6dc238675c137d41f4 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:00:34 +0200 Subject: [PATCH 4/7] validate table name in addPurgingTable Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/gc/tablegc.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/vt/vttablet/tabletserver/gc/tablegc.go b/go/vt/vttablet/tabletserver/gc/tablegc.go index 8658d7c3a3b..c0adc2dc252 100644 --- a/go/vt/vttablet/tabletserver/gc/tablegc.go +++ b/go/vt/vttablet/tabletserver/gc/tablegc.go @@ -608,6 +608,16 @@ func (collector *TableGC) addPurgingTable(tableName string) (added bool) { // so we don't populate the purgingTables map. return false } + isGCTable, state, _, _, err := schema.AnalyzeGCTableName(tableName) + if err != nil { + return false + } + if !isGCTable { + return false + } + if state != schema.PurgeTableGCState { + return false + } collector.purgeMutex.Lock() defer collector.purgeMutex.Unlock() From 3f0d8b0c1350ac802e84b51a88d14b0edd3de7b0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:00:47 +0200 Subject: [PATCH 5/7] test new formats Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../vttablet/tabletserver/gc/tablegc_test.go | 186 ++++++++++++++++-- 1 file changed, 173 insertions(+), 13 deletions(-) diff --git a/go/vt/vttablet/tabletserver/gc/tablegc_test.go b/go/vt/vttablet/tabletserver/gc/tablegc_test.go index 446f6e6ff85..47012b26d87 100644 --- a/go/vt/vttablet/tabletserver/gc/tablegc_test.go +++ b/go/vt/vttablet/tabletserver/gc/tablegc_test.go @@ -29,15 +29,18 @@ import ( func TestNextTableToPurge(t *testing.T) { tt := []struct { + name string tables []string next string ok bool }{ { + name: "empty", tables: []string{}, ok: false, }, { + name: "first", tables: []string{ "_vt_PURGE_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", "_vt_PURGE_2ace8bcef73211ea87e9f875a4d24e90_20200915120411", @@ -48,6 +51,7 @@ func TestNextTableToPurge(t *testing.T) { ok: true, }, { + name: "mid", tables: []string{ "_vt_PURGE_2ace8bcef73211ea87e9f875a4d24e90_20200915120411", "_vt_PURGE_3ace8bcef73211ea87e9f875a4d24e90_20200915120412", @@ -57,19 +61,69 @@ func TestNextTableToPurge(t *testing.T) { next: "_vt_PURGE_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", ok: true, }, + { + name: "none", + tables: []string{ + "_vt_HOLD_2ace8bcef73211ea87e9f875a4d24e90_20200915120411", + "_vt_EVAC_3ace8bcef73211ea87e9f875a4d24e90_20200915120412", + "_vt_EVAC_6ace8bcef73211ea87e9f875a4d24e90_20200915120410", + "_vt_DROP_4ace8bcef73211ea87e9f875a4d24e90_20200915120413", + }, + next: "", + ok: false, + }, + { + name: "first, new format", + tables: []string{ + "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_prg_2ace8bcef73211ea87e9f875a4d24e90_20200915120411_", + "_vt_prg_3ace8bcef73211ea87e9f875a4d24e90_20200915120412_", + "_vt_prg_4ace8bcef73211ea87e9f875a4d24e90_20200915120413_", + }, + next: "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + ok: true, + }, + { + name: "mid, new format", + tables: []string{ + "_vt_prg_2ace8bcef73211ea87e9f875a4d24e90_20200915120411_", + "_vt_prg_3ace8bcef73211ea87e9f875a4d24e90_20200915120412_", + "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_prg_4ace8bcef73211ea87e9f875a4d24e90_20200915120413_", + }, + next: "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + ok: true, + }, + { + name: "none, new format", + tables: []string{ + "_vt_hld_2ace8bcef73211ea87e9f875a4d24e90_20200915120411_", + "_vt_evc_3ace8bcef73211ea87e9f875a4d24e90_20200915120412_", + "_vt_evc_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + "_vt_drp_4ace8bcef73211ea87e9f875a4d24e90_20200915120413_", + "_vt_prg_4ace8bcef73211ea87e9f875a4d24e90_20200915999999_", + }, + next: "", + ok: false, + }, } for _, ts := range tt { - collector := &TableGC{ - purgingTables: make(map[string]bool), - } - for _, table := range ts.tables { - collector.purgingTables[table] = true - } - next, ok := collector.nextTableToPurge() - assert.Equal(t, ts.ok, ok) - if ok { - assert.Equal(t, ts.next, next) - } + t.Run(ts.name, func(t *testing.T) { + collector := &TableGC{ + purgingTables: make(map[string]bool), + } + var err error + collector.lifecycleStates, err = schema.ParseGCLifecycle("hold,purge,evac,drop") + assert.NoError(t, err) + for _, table := range ts.tables { + collector.addPurgingTable(table) + } + next, ok := collector.nextTableToPurge() + assert.Equal(t, ts.ok, ok) + if ok { + assert.Equal(t, ts.next, next) + } + }) } } @@ -171,6 +225,13 @@ func TestShouldTransitionTable(t *testing.T) { uuid: "6ace8bcef73211ea87e9f875a4d24e90", shouldTransition: true, }, + { + name: "purge, old timestamp, new format", + table: "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", + state: schema.PurgeTableGCState, + uuid: "6ace8bcef73211ea87e9f875a4d24e90", + shouldTransition: true, + }, { name: "no purge, future timestamp", table: "_vt_PURGE_6ace8bcef73211ea87e9f875a4d24e90_29990915120410", @@ -178,6 +239,13 @@ func TestShouldTransitionTable(t *testing.T) { uuid: "6ace8bcef73211ea87e9f875a4d24e90", shouldTransition: false, }, + { + name: "no purge, future timestamp, new format", + table: "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_29990915120410_", + state: schema.PurgeTableGCState, + uuid: "6ace8bcef73211ea87e9f875a4d24e90", + shouldTransition: false, + }, { name: "no purge, PURGE not handled state", table: "_vt_PURGE_6ace8bcef73211ea87e9f875a4d24e90_29990915120410", @@ -186,6 +254,14 @@ func TestShouldTransitionTable(t *testing.T) { handledStates: "hold,evac", // no PURGE shouldTransition: true, }, + { + name: "no purge, PURGE not handled state, new format", + table: "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_29990915120410_", + state: schema.PurgeTableGCState, + uuid: "6ace8bcef73211ea87e9f875a4d24e90", + handledStates: "hold,evac", // no PURGE + shouldTransition: true, + }, { name: "no drop, future timestamp", table: "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_29990915120410", @@ -193,6 +269,13 @@ func TestShouldTransitionTable(t *testing.T) { uuid: "6ace8bcef73211ea87e9f875a4d24e90", shouldTransition: false, }, + { + name: "no drop, future timestamp, new format", + table: "_vt_drp_6ace8bcef73211ea87e9f875a4d24e90_29990915120410_", + state: schema.DropTableGCState, + uuid: "6ace8bcef73211ea87e9f875a4d24e90", + shouldTransition: false, + }, { name: "drop, old timestamp", table: "_vt_DROP_6ace8bcef73211ea87e9f875a4d24e90_20090915120410", @@ -200,6 +283,13 @@ func TestShouldTransitionTable(t *testing.T) { uuid: "6ace8bcef73211ea87e9f875a4d24e90", shouldTransition: true, }, + { + name: "drop, old timestamp, new format", + table: "_vt_drp_6ace8bcef73211ea87e9f875a4d24e90_20090915120410_", + state: schema.DropTableGCState, + uuid: "6ace8bcef73211ea87e9f875a4d24e90", + shouldTransition: true, + }, { name: "no evac, future timestamp", table: "_vt_EVAC_6ace8bcef73211ea87e9f875a4d24e90_29990915120410", @@ -207,6 +297,13 @@ func TestShouldTransitionTable(t *testing.T) { uuid: "6ace8bcef73211ea87e9f875a4d24e90", shouldTransition: false, }, + { + name: "no evac, future timestamp, new format", + table: "_vt_evc_6ace8bcef73211ea87e9f875a4d24e90_29990915120410_", + state: schema.EvacTableGCState, + uuid: "6ace8bcef73211ea87e9f875a4d24e90", + shouldTransition: false, + }, { name: "no hold, HOLD not handled state", table: "_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_29990915120410", @@ -214,6 +311,13 @@ func TestShouldTransitionTable(t *testing.T) { uuid: "6ace8bcef73211ea87e9f875a4d24e90", shouldTransition: true, }, + { + name: "no hold, HOLD not handled state, new format", + table: "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_29990915120410_", + state: schema.HoldTableGCState, + uuid: "6ace8bcef73211ea87e9f875a4d24e90", + shouldTransition: true, + }, { name: "hold, future timestamp", table: "_vt_HOLD_6ace8bcef73211ea87e9f875a4d24e90_29990915120410", @@ -222,6 +326,14 @@ func TestShouldTransitionTable(t *testing.T) { handledStates: "hold,purge,evac,drop", shouldTransition: false, }, + { + name: "hold, future timestamp, new format", + table: "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_29990915120410_", + state: schema.HoldTableGCState, + uuid: "6ace8bcef73211ea87e9f875a4d24e90", + handledStates: "hold,purge,evac,drop", + shouldTransition: false, + }, { name: "not a GC table", table: "_vt_SOMETHING_6ace8bcef73211ea87e9f875a4d24e90_29990915120410", @@ -229,6 +341,13 @@ func TestShouldTransitionTable(t *testing.T) { uuid: "", shouldTransition: false, }, + { + name: "invalid new format", + table: "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_29990915999999_", + state: "", + uuid: "", + shouldTransition: false, + }, } for _, ts := range tt { t.Run(ts.name, func(t *testing.T) { @@ -268,35 +387,70 @@ func TestCheckTables(t *testing.T) { tableName: "_vt_something_that_isnt_a_gc_table", isBaseTable: true, }, + { + tableName: "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_29990915999999_", + isBaseTable: true, + }, { tableName: "_vt_HOLD_11111111111111111111111111111111_20990920093324", // 2099 is in the far future isBaseTable: true, }, + { + tableName: "_vt_hld_11111111111111111111111111111111_20990920093324_", // 2099 is in the far future + isBaseTable: true, + }, { tableName: "_vt_HOLD_22222222222222222222222222222222_20200920093324", isBaseTable: true, }, + { + tableName: "_vt_hld_22222222222222222222222222222222_20200920093324_", + isBaseTable: true, + }, { tableName: "_vt_DROP_33333333333333333333333333333333_20200919083451", isBaseTable: true, }, + { + tableName: "_vt_drp_33333333333333333333333333333333_20200919083451_", + isBaseTable: true, + }, { tableName: "_vt_DROP_44444444444444444444444444444444_20200919083451", isBaseTable: false, }, + { + tableName: "_vt_drp_44444444444444444444444444444444_20200919083451_", + isBaseTable: false, + }, } - // one gcTable above is irrelevant, does not have a GC table name + expectResponses := len(gcTables) + // one gcTable above is irrelevant: it does not have a GC table name + expectResponses = expectResponses - 1 // one will not transition: its date is 2099 - expectResponses := len(gcTables) - 2 + expectResponses = expectResponses - 1 + // one gcTable above is irrelevant: it has an invalid new format timestamp + expectResponses = expectResponses - 1 + // one will not transition: its date is 2099 in new format + expectResponses = expectResponses - 1 + expectDropTables := []*gcTable{ { tableName: "_vt_DROP_33333333333333333333333333333333_20200919083451", isBaseTable: true, }, + { + tableName: "_vt_drp_33333333333333333333333333333333_20200919083451_", + isBaseTable: true, + }, { tableName: "_vt_DROP_44444444444444444444444444444444_20200919083451", isBaseTable: false, }, + { + tableName: "_vt_drp_44444444444444444444444444444444_20200919083451_", + isBaseTable: false, + }, } expectTransitionRequests := []*transitionRequest{ { @@ -305,6 +459,12 @@ func TestCheckTables(t *testing.T) { toGCState: schema.PurgeTableGCState, uuid: "22222222222222222222222222222222", }, + { + fromTableName: "_vt_hld_22222222222222222222222222222222_20200920093324_", + isBaseTable: true, + toGCState: schema.PurgeTableGCState, + uuid: "22222222222222222222222222222222", + }, } ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) From 41fd5526ee43900923adda936b32af23581108a9 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:01:07 +0200 Subject: [PATCH 6/7] test new formats Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletmanager/tablegc/tablegc_test.go | 191 +++++++++++------- 1 file changed, 114 insertions(+), 77 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index c6f7253c791..23fa2e4a54c 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -20,6 +20,7 @@ import ( "flag" "fmt" "os" + "strings" "testing" "time" @@ -158,10 +159,17 @@ func populateTable(t *testing.T) { } // tableExists sees that a given table exists in MySQL -func tableExists(tableExpr string) (exists bool, tableName string, err error) { - query := `select table_name as table_name from information_schema.tables where table_schema=database() and table_name like '%a'` - parsed := sqlparser.BuildParsedQuery(query, tableExpr) - rs, err := primaryTablet.VttabletProcess.QueryTablet(parsed.Query, keyspaceName, true) +func tableExists(exprs ...string) (exists bool, tableName string, err error) { + if len(exprs) == 0 { + return false, "", fmt.Errorf("empty table list") + } + var clauses []string + for _, expr := range exprs { + clauses = append(clauses, fmt.Sprintf("table_name like '%s'", expr)) + } + clause := strings.Join(clauses, " or ") + query := fmt.Sprintf(`select table_name as table_name from information_schema.tables where table_schema=database() and (%s)`, clause) + rs, err := primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) if err != nil { return false, "", err } @@ -220,23 +228,29 @@ func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableG for _, state := range states { expectTableToExist := true searchExpr := "" + searchExpr2 := "" switch state { case schema.HoldTableGCState: searchExpr = `\_vt\_HOLD\_%` + searchExpr2 = `\_vt\_hld\_%` case schema.PurgeTableGCState: searchExpr = `\_vt\_PURGE\_%` + searchExpr2 = `\_vt\_prg\_%` case schema.EvacTableGCState: searchExpr = `\_vt\_EVAC\_%` + searchExpr2 = `\_vt\_evc\_%` case schema.DropTableGCState: searchExpr = `\_vt\_DROP\_%` + searchExpr2 = `\_vt\_drp\_%` case schema.TableDroppedGCState: searchExpr = `\_vt\_%` + searchExpr2 = `\_vt\_%` expectTableToExist = false default: t.Log("Unknown state") t.Fail() } - exists, tableName, err := tableExists(searchExpr) + exists, tableName, err := tableExists(searchExpr, searchExpr2) require.NoError(t, err) if exists { @@ -279,102 +293,125 @@ func TestPopulateTable(t *testing.T) { validateTableDoesNotExist(t, "no_such_table") } +func generateRenameStatement(newFormat bool, fromTableName string, state schema.TableGCState, tm time.Time) (statement string, toTableName string, err error) { + if newFormat { + return schema.GenerateRenameStatementNewFormat(fromTableName, state, tm) + } + return schema.GenerateRenameStatement(fromTableName, state, tm) +} + func TestHold(t *testing.T) { - populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("t1", schema.HoldTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) - assert.NoError(t, err) + for _, newNameFormat := range []bool{false, true} { + t.Run(fmt.Sprintf("new format=%t", newNameFormat), func(t *testing.T) { + populateTable(t) + query, tableName, err := generateRenameStatement(newNameFormat, "t1", schema.HoldTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) + assert.NoError(t, err) - _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) - assert.NoError(t, err) + _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) + assert.NoError(t, err) - validateTableDoesNotExist(t, "t1") - validateTableExists(t, tableName) + validateTableDoesNotExist(t, "t1") + validateTableExists(t, tableName) - time.Sleep(tableTransitionExpiration / 2) - { - // Table was created with +10s timestamp, so it should still exist - validateTableExists(t, tableName) + time.Sleep(tableTransitionExpiration / 2) + { + // Table was created with +10s timestamp, so it should still exist + validateTableExists(t, tableName) - checkTableRows(t, tableName, 1024) - } + checkTableRows(t, tableName, 1024) + } - time.Sleep(tableTransitionExpiration) - // We're now both beyond table's timestamp as well as a tableGC interval - validateTableDoesNotExist(t, tableName) - if fastDropTable { - validateAnyState(t, -1, schema.DropTableGCState, schema.TableDroppedGCState) - } else { - validateAnyState(t, -1, schema.PurgeTableGCState, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) + time.Sleep(tableTransitionExpiration) + // We're now both beyond table's timestamp as well as a tableGC interval + validateTableDoesNotExist(t, tableName) + if fastDropTable { + validateAnyState(t, -1, schema.DropTableGCState, schema.TableDroppedGCState) + } else { + validateAnyState(t, -1, schema.PurgeTableGCState, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) + } + }) } } func TestEvac(t *testing.T) { - populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("t1", schema.EvacTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) - assert.NoError(t, err) - - _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) - assert.NoError(t, err) - - validateTableDoesNotExist(t, "t1") + for _, newNameFormat := range []bool{false, true} { + t.Run(fmt.Sprintf("new format=%t", newNameFormat), func(t *testing.T) { + populateTable(t) + query, tableName, err := generateRenameStatement(newNameFormat, "t1", schema.EvacTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) + assert.NoError(t, err) + + _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) + assert.NoError(t, err) + + validateTableDoesNotExist(t, "t1") + + time.Sleep(tableTransitionExpiration / 2) + { + // Table was created with +10s timestamp, so it should still exist + if fastDropTable { + // EVAC state is skipped in mysql 8.0.23 and beyond + validateTableDoesNotExist(t, tableName) + } else { + validateTableExists(t, tableName) + checkTableRows(t, tableName, 1024) + } + } - time.Sleep(tableTransitionExpiration / 2) - { - // Table was created with +10s timestamp, so it should still exist - if fastDropTable { - // EVAC state is skipped in mysql 8.0.23 and beyond + time.Sleep(tableTransitionExpiration) + // We're now both beyond table's timestamp as well as a tableGC interval validateTableDoesNotExist(t, tableName) - } else { - validateTableExists(t, tableName) - checkTableRows(t, tableName, 1024) - } + // Table should be renamed as _vt_DROP_... and then dropped! + validateAnyState(t, 0, schema.DropTableGCState, schema.TableDroppedGCState) + }) } - - time.Sleep(tableTransitionExpiration) - // We're now both beyond table's timestamp as well as a tableGC interval - validateTableDoesNotExist(t, tableName) - // Table should be renamed as _vt_DROP_... and then dropped! - validateAnyState(t, 0, schema.DropTableGCState, schema.TableDroppedGCState) } func TestDrop(t *testing.T) { - populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("t1", schema.DropTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) - assert.NoError(t, err) + for _, newNameFormat := range []bool{false, true} { + t.Run(fmt.Sprintf("new format=%t", newNameFormat), func(t *testing.T) { + populateTable(t) + query, tableName, err := generateRenameStatement(newNameFormat, "t1", schema.DropTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) + assert.NoError(t, err) - _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) - assert.NoError(t, err) + _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) + assert.NoError(t, err) - validateTableDoesNotExist(t, "t1") + validateTableDoesNotExist(t, "t1") - time.Sleep(tableTransitionExpiration) - time.Sleep(2 * gcCheckInterval) - // We're now both beyond table's timestamp as well as a tableGC interval - validateTableDoesNotExist(t, tableName) + time.Sleep(tableTransitionExpiration) + time.Sleep(2 * gcCheckInterval) + // We're now both beyond table's timestamp as well as a tableGC interval + validateTableDoesNotExist(t, tableName) + }) + } } func TestPurge(t *testing.T) { - populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("t1", schema.PurgeTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) - require.NoError(t, err) + for _, newNameFormat := range []bool{false, true} { + t.Run(fmt.Sprintf("new format=%t", newNameFormat), func(t *testing.T) { + populateTable(t) + query, tableName, err := generateRenameStatement(newNameFormat, "t1", schema.PurgeTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) + require.NoError(t, err) - _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) - require.NoError(t, err) + _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) + require.NoError(t, err) - validateTableDoesNotExist(t, "t1") - if !fastDropTable { - validateTableExists(t, tableName) - checkTableRows(t, tableName, 1024) - } - if !fastDropTable { - time.Sleep(5 * gcPurgeCheckInterval) // wait for table to be purged - } - validateTableDoesNotExist(t, tableName) // whether purged or not, table should at some point transition to next state - if fastDropTable { - // if MySQL supports fast DROP TABLE, TableGC completely skips the PURGE state. Rows are not purged. - validateAnyState(t, 1024, schema.DropTableGCState, schema.TableDroppedGCState) - } else { - validateAnyState(t, 0, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) + validateTableDoesNotExist(t, "t1") + if !fastDropTable { + validateTableExists(t, tableName) + checkTableRows(t, tableName, 1024) + } + if !fastDropTable { + time.Sleep(5 * gcPurgeCheckInterval) // wait for table to be purged + } + validateTableDoesNotExist(t, tableName) // whether purged or not, table should at some point transition to next state + if fastDropTable { + // if MySQL supports fast DROP TABLE, TableGC completely skips the PURGE state. Rows are not purged. + validateAnyState(t, 1024, schema.DropTableGCState, schema.TableDroppedGCState) + } else { + validateAnyState(t, 0, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) + } + }) } } From b9cf862f74fbacedee713b16ed01807b56d181e8 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 7 Jan 2024 09:37:37 +0200 Subject: [PATCH 7/7] simplify code; adding comments Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/schema/name.go | 1 + go/vt/schema/online_ddl.go | 4 ++-- go/vt/schema/tablegc.go | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go/vt/schema/name.go b/go/vt/schema/name.go index 4e8ec14db14..9a3f038c477 100644 --- a/go/vt/schema/name.go +++ b/go/vt/schema/name.go @@ -33,6 +33,7 @@ const ( ) var ( + // internalTableNameRegexp parses new intrnal table name format, e.g. _vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_ internalTableNameRegexp = regexp.MustCompile(InternalTableNameExpression) ) diff --git a/go/vt/schema/online_ddl.go b/go/vt/schema/online_ddl.go index 66d5bd2982c..670763a69f3 100644 --- a/go/vt/schema/online_ddl.go +++ b/go/vt/schema/online_ddl.go @@ -471,10 +471,10 @@ func OnlineDDLToGCUUID(uuid string) string { // by pt-online-schema-change. // There is no guarantee that the tables _was indeed_ generated by an online DDL flow. func IsOnlineDDLTableName(tableName string) bool { - // Try new naming format (e.g. `_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_`): + // Try new naming format (e.g. `_vt_vrp_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_`): // The new naming format is accepted in v19, and actually _used_ in v20 if isInternal, hint, _, _, _ := AnalyzeInternalTableName(tableName); isInternal { - return isInternal && onlineDDLInternalTableHintsMap[hint] + return onlineDDLInternalTableHintsMap[hint] } if onlineDDLGeneratedTableNameRegexp.MatchString(tableName) { diff --git a/go/vt/schema/tablegc.go b/go/vt/schema/tablegc.go index 81aaee15957..2f102085f00 100644 --- a/go/vt/schema/tablegc.go +++ b/go/vt/schema/tablegc.go @@ -45,7 +45,8 @@ const ( ) const ( - GCTableNameExpression string = `^_vt_(HOLD|PURGE|EVAC|DROP)_([0-f]{32})_([0-9]{14})$` + GCTableNameExpression string = `^_vt_(HOLD|PURGE|EVAC|DROP)_([0-f]{32})_([0-9]{14})$` + // NewGCTableNameExpression parses new intrnal table name format, e.g. _vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_ NewGCTableNameExpression string = `^_vt_(hld|prg|evc|drp)_([0-f]{32})_([0-9]{14})_$` )