From 0885dffca6baabafbaf5a370b4cf70652a4b752c Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 24 Nov 2023 13:46:01 -0800 Subject: [PATCH 01/13] update_method_backupShard Signed-off-by: Jun Wang --- go/vt/vtctl/grpcvtctldserver/server.go | 28 ++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 21c4dc272f6..6b56847daaf 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io" + "math/rand" "net/http" "path/filepath" "runtime/debug" @@ -430,9 +431,32 @@ func (s *VtctldServer) BackupShard(req *vtctldatapb.BackupShardRequest, stream v span.Annotate("concurrency", req.Concurrency) span.Annotate("incremental_from_pos", req.IncrementalFromPos) - tablets, stats, err := reparentutil.ShardReplicationStatuses(ctx, s.ts, s.tmc, req.Keyspace, req.Shard) + shardTablets, stats, err := reparentutil.ShardReplicationStatuses(ctx, s.ts, s.tmc, req.Keyspace, req.Shard) + // Shuffle shardTablets to avoid items in a fixed order + rand.Shuffle(len(shardTablets), func(i, j int) { + shardTablets[i], shardTablets[j] = shardTablets[j], shardTablets[i] + }) + + var tablets []*topo.TabletInfo + // Instead of return on err directly, count total errors and compare with len(stats) if err != nil { - return err + nilStatIndex, errorCount := 0, 0 + for i, stat := range stats { + if stat == nil { + // Possible of multiple errors but only catch the last error index in stats + nilStatIndex = i + errorCount++ + } + } + // Only return err when errors on all vttablets + if errorCount == len(stats) { + return err + } + for i, shardTablet := range shardTablets { + if i != nilStatIndex { + tablets = append(tablets, shardTablet) + } + } } var ( From 7714d0ad2dcc192d65155cbc537f38607440ff4b Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 24 Nov 2023 14:32:14 -0800 Subject: [PATCH 02/13] check if stat not nil Signed-off-by: Jun Wang --- go/vt/vtctl/grpcvtctldserver/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 6b56847daaf..eda9531c73a 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -442,7 +442,7 @@ func (s *VtctldServer) BackupShard(req *vtctldatapb.BackupShardRequest, stream v if err != nil { nilStatIndex, errorCount := 0, 0 for i, stat := range stats { - if stat == nil { + if stat != nil { // Possible of multiple errors but only catch the last error index in stats nilStatIndex = i errorCount++ From 403caadc5fd83a8865358f8fdbb4654ad98f836a Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 24 Nov 2023 15:04:56 -0800 Subject: [PATCH 03/13] skip checking stat of Primary Signed-off-by: Jun Wang --- go/vt/vtctl/grpcvtctldserver/server.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index eda9531c73a..d2de986fdcd 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -442,14 +442,18 @@ func (s *VtctldServer) BackupShard(req *vtctldatapb.BackupShardRequest, stream v if err != nil { nilStatIndex, errorCount := 0, 0 for i, stat := range stats { - if stat != nil { + // Skip Primary + if stat != nil && stat.Position != "" { + continue + } + if stat == nil { // Possible of multiple errors but only catch the last error index in stats nilStatIndex = i errorCount++ } } - // Only return err when errors on all vttablets - if errorCount == len(stats) { + // Only return err when all Non-Primary have errors + if errorCount == len(stats)-1 { return err } for i, shardTablet := range shardTablets { From ae91011f18e6d5198a8eb92b4fcbaf8f9c1e64cd Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 24 Nov 2023 15:20:27 -0800 Subject: [PATCH 04/13] update_method_backupShard Signed-off-by: Jun Wang --- go/vt/vtctl/grpcvtctldserver/server.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index d2de986fdcd..b4683b422d4 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -438,9 +438,9 @@ func (s *VtctldServer) BackupShard(req *vtctldatapb.BackupShardRequest, stream v }) var tablets []*topo.TabletInfo + nilStatIndex, errorCount := 0, 0 // Instead of return on err directly, count total errors and compare with len(stats) if err != nil { - nilStatIndex, errorCount := 0, 0 for i, stat := range stats { // Skip Primary if stat != nil && stat.Position != "" { @@ -456,6 +456,9 @@ func (s *VtctldServer) BackupShard(req *vtctldatapb.BackupShardRequest, stream v if errorCount == len(stats)-1 { return err } + } + + if errorCount != 0 { for i, shardTablet := range shardTablets { if i != nilStatIndex { tablets = append(tablets, shardTablet) From 2b4bc65f2966bf9e64e8091af33b04f8f19bf421 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 24 Nov 2023 15:33:26 -0800 Subject: [PATCH 05/13] update_method_backupShard Signed-off-by: Jun Wang --- go/vt/vtctl/grpcvtctldserver/server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index b4683b422d4..8bf21ac45d6 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -464,6 +464,8 @@ func (s *VtctldServer) BackupShard(req *vtctldatapb.BackupShardRequest, stream v tablets = append(tablets, shardTablet) } } + } else { + tablets = shardTablets } var ( From 6da34b3a38c60567400301664b86dacbdaa10963 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 24 Nov 2023 23:00:53 -0800 Subject: [PATCH 06/13] update_method_backupShard Signed-off-by: Jun Wang --- go/vt/vtctl/grpcvtctldserver/server.go | 32 ++++++-------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 8bf21ac45d6..1b8d2a01ded 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "io" - "math/rand" "net/http" "path/filepath" "runtime/debug" @@ -432,38 +431,21 @@ func (s *VtctldServer) BackupShard(req *vtctldatapb.BackupShardRequest, stream v span.Annotate("incremental_from_pos", req.IncrementalFromPos) shardTablets, stats, err := reparentutil.ShardReplicationStatuses(ctx, s.ts, s.tmc, req.Keyspace, req.Shard) - // Shuffle shardTablets to avoid items in a fixed order - rand.Shuffle(len(shardTablets), func(i, j int) { - shardTablets[i], shardTablets[j] = shardTablets[j], shardTablets[i] - }) var tablets []*topo.TabletInfo - nilStatIndex, errorCount := 0, 0 - // Instead of return on err directly, count total errors and compare with len(stats) + // Instead of return on err directly, only return when no healthy tablets at all if err != nil { for i, stat := range stats { - // Skip Primary - if stat != nil && stat.Position != "" { - continue - } - if stat == nil { - // Possible of multiple errors but only catch the last error index in stats - nilStatIndex = i - errorCount++ + // shardTablets[i] and stats[i] is 1:1 mapping + // Healthy shardTablets[i] will be added to tablets + if stat != nil { + tablets = append(tablets, shardTablets[i]) } } - // Only return err when all Non-Primary have errors - if errorCount == len(stats)-1 { + // Only return err when all tablets have errors + if len(tablets) == 0 { return err } - } - - if errorCount != 0 { - for i, shardTablet := range shardTablets { - if i != nilStatIndex { - tablets = append(tablets, shardTablet) - } - } } else { tablets = shardTablets } From 21993175dd6a0cd8b21b96bce096eeb5128fe6fc Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Mon, 27 Nov 2023 10:12:31 -0800 Subject: [PATCH 07/13] always include TabletType_PRIMARY in returned tablets Signed-off-by: Jun Wang --- go/vt/vtctl/grpcvtctldserver/server.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 1b8d2a01ded..09a88b4fa24 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -436,13 +436,18 @@ func (s *VtctldServer) BackupShard(req *vtctldatapb.BackupShardRequest, stream v // Instead of return on err directly, only return when no healthy tablets at all if err != nil { for i, stat := range stats { + // Always include TabletType_PRIMARY + if shardTablets[i].Type == topodatapb.TabletType_PRIMARY { + tablets = append(tablets, shardTablets[i]) + continue + } // shardTablets[i] and stats[i] is 1:1 mapping // Healthy shardTablets[i] will be added to tablets if stat != nil { tablets = append(tablets, shardTablets[i]) } } - // Only return err when all tablets have errors + // Only return err when no usable tablet if len(tablets) == 0 { return err } From 94b9d2185edcfea2774431b29e1fbc624d1b18b2 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 28 Nov 2023 12:19:24 -0800 Subject: [PATCH 08/13] create func GetBackupCandidates Signed-off-by: Jun Wang --- go/vt/vtctl/grpcvtctldserver/server.go | 20 +++----------------- go/vt/vtctl/reparentutil/util.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 09a88b4fa24..caa99d5e8c8 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -430,29 +430,15 @@ func (s *VtctldServer) BackupShard(req *vtctldatapb.BackupShardRequest, stream v span.Annotate("concurrency", req.Concurrency) span.Annotate("incremental_from_pos", req.IncrementalFromPos) - shardTablets, stats, err := reparentutil.ShardReplicationStatuses(ctx, s.ts, s.tmc, req.Keyspace, req.Shard) + tablets, stats, err := reparentutil.ShardReplicationStatuses(ctx, s.ts, s.tmc, req.Keyspace, req.Shard) - var tablets []*topo.TabletInfo - // Instead of return on err directly, only return when no healthy tablets at all + // Instead of return on err directly, only return err when no tablets for backup at all if err != nil { - for i, stat := range stats { - // Always include TabletType_PRIMARY - if shardTablets[i].Type == topodatapb.TabletType_PRIMARY { - tablets = append(tablets, shardTablets[i]) - continue - } - // shardTablets[i] and stats[i] is 1:1 mapping - // Healthy shardTablets[i] will be added to tablets - if stat != nil { - tablets = append(tablets, shardTablets[i]) - } - } + tablets = reparentutil.GetBackupCandidates(tablets, stats) // Only return err when no usable tablet if len(tablets) == 0 { return err } - } else { - tablets = shardTablets } var ( diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index cfde8f34508..a3132d46ba1 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -343,3 +343,20 @@ func waitForCatchUp( } return nil } + +// GetBackupCandidates is used to get a list of healthy tablets for backup +func GetBackupCandidates(tablets []*topo.TabletInfo, stats []*replicationdatapb.Status) (res []*topo.TabletInfo) { + for i, stat := range stats { + // Always include TabletType_PRIMARY + if tablets[i].Type == topodatapb.TabletType_PRIMARY { + res = append(tablets, tablets[i]) + continue + } + // shardTablets[i] and stats[i] is 1:1 mapping + // Healthy shardTablets[i] will be added to tablets + if stat != nil { + res = append(tablets, tablets[i]) + } + } + return res +} From d8463160e2e0a1631010640056f81285ccc140f3 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 28 Nov 2023 13:22:10 -0800 Subject: [PATCH 09/13] update func GetBackupCandidates Signed-off-by: Jun Wang --- go/vt/vtctl/reparentutil/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index a3132d46ba1..2f6f846e793 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -349,13 +349,13 @@ func GetBackupCandidates(tablets []*topo.TabletInfo, stats []*replicationdatapb. for i, stat := range stats { // Always include TabletType_PRIMARY if tablets[i].Type == topodatapb.TabletType_PRIMARY { - res = append(tablets, tablets[i]) + res = append(res, tablets[i]) continue } // shardTablets[i] and stats[i] is 1:1 mapping // Healthy shardTablets[i] will be added to tablets if stat != nil { - res = append(tablets, tablets[i]) + res = append(res, tablets[i]) } } return res From ddad61a8839920dcbc0d2c2236b73111ebef6ccd Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 28 Nov 2023 14:41:48 -0800 Subject: [PATCH 10/13] update func GetBackupCandidates Signed-off-by: Jun Wang --- go/vt/vtctl/reparentutil/util.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 2f6f846e793..0dca8627e3b 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -346,6 +346,16 @@ func waitForCatchUp( // GetBackupCandidates is used to get a list of healthy tablets for backup func GetBackupCandidates(tablets []*topo.TabletInfo, stats []*replicationdatapb.Status) (res []*topo.TabletInfo) { + // In case of all tablets has nil stats, return TabletType_PRIMARY + if len(stats) == 0 { + for _, tablet := range tablets { + if tablet.Type == topodatapb.TabletType_PRIMARY { + res = append(res, tablet) + break + } + } + return res + } for i, stat := range stats { // Always include TabletType_PRIMARY if tablets[i].Type == topodatapb.TabletType_PRIMARY { From d08edca567073326854da892e324598c6c249a11 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 28 Nov 2023 15:16:45 -0800 Subject: [PATCH 11/13] update func GetBackupCandidates and add func Test_GetBackupCandidates Signed-off-by: Jun Wang --- go/vt/vtctl/reparentutil/util.go | 10 --- go/vt/vtctl/reparentutil/util_test.go | 102 ++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 10 deletions(-) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 0dca8627e3b..2f6f846e793 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -346,16 +346,6 @@ func waitForCatchUp( // GetBackupCandidates is used to get a list of healthy tablets for backup func GetBackupCandidates(tablets []*topo.TabletInfo, stats []*replicationdatapb.Status) (res []*topo.TabletInfo) { - // In case of all tablets has nil stats, return TabletType_PRIMARY - if len(stats) == 0 { - for _, tablet := range tablets { - if tablet.Type == topodatapb.TabletType_PRIMARY { - res = append(res, tablet) - break - } - } - return res - } for i, stat := range stats { // Always include TabletType_PRIMARY if tablets[i].Type == topodatapb.TabletType_PRIMARY { diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index a9e6274d490..137f09eec99 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1218,3 +1218,105 @@ func Test_getTabletsWithPromotionRules(t *testing.T) { }) } } + +func Test_GetBackupCandidates(t *testing.T) { + var ( + primaryTablet = &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 1, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + } + replicaTablet = &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 2, + }, + Type: topodatapb.TabletType_REPLICA, + }, + } + rdonlyTablet = &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 3, + }, + Type: topodatapb.TabletType_RDONLY, + }, + } + spareTablet = &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 4, + }, + Type: topodatapb.TabletType_SPARE, + }, + } + ) + tests := []struct { + name string + in []*topo.TabletInfo + expected []*topo.TabletInfo + status []*replicationdatapb.Status + }{ + { + name: "one primary tablet with status", + in: []*topo.TabletInfo{primaryTablet}, + expected: []*topo.TabletInfo{primaryTablet}, + status: []*replicationdatapb.Status{{}}, + }, + { + name: "one primary tablet with no status", + in: []*topo.TabletInfo{primaryTablet}, + expected: []*topo.TabletInfo{primaryTablet}, + status: []*replicationdatapb.Status{nil}, + }, + { + name: "4 tablets with no status", + in: []*topo.TabletInfo{primaryTablet, replicaTablet, rdonlyTablet, spareTablet}, + expected: []*topo.TabletInfo{primaryTablet}, + status: []*replicationdatapb.Status{nil, nil, nil, nil}, + }, + { + name: "4 tablets with full status", + in: []*topo.TabletInfo{primaryTablet, replicaTablet, rdonlyTablet, spareTablet}, + expected: []*topo.TabletInfo{primaryTablet, replicaTablet, rdonlyTablet, spareTablet}, + status: []*replicationdatapb.Status{{}, {}, {}, {}}, + }, + { + name: "4 tablets with no primaryTablet status", + in: []*topo.TabletInfo{primaryTablet, replicaTablet, rdonlyTablet, spareTablet}, + expected: []*topo.TabletInfo{primaryTablet, replicaTablet, rdonlyTablet, spareTablet}, + status: []*replicationdatapb.Status{nil, {}, {}, {}}, + }, + { + name: "4 tablets with no replicaTablet status", + in: []*topo.TabletInfo{primaryTablet, replicaTablet, rdonlyTablet, spareTablet}, + expected: []*topo.TabletInfo{primaryTablet, rdonlyTablet, spareTablet}, + status: []*replicationdatapb.Status{{}, nil, {}, {}}, + }, + { + name: "4 tablets with no rdonlyTablet status", + in: []*topo.TabletInfo{primaryTablet, replicaTablet, rdonlyTablet, spareTablet}, + expected: []*topo.TabletInfo{primaryTablet, replicaTablet, spareTablet}, + status: []*replicationdatapb.Status{{}, {}, nil, {}}, + }, + { + name: "4 tablets with no spareTablet status", + in: []*topo.TabletInfo{primaryTablet, replicaTablet, rdonlyTablet, spareTablet}, + expected: []*topo.TabletInfo{primaryTablet, replicaTablet, rdonlyTablet}, + status: []*replicationdatapb.Status{{}, {}, {}, nil}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res := GetBackupCandidates(tt.in, tt.status) + require.EqualValues(t, tt.expected, res) + }) + } +} From 370a3cfbe10f07772b4f4a21b0585ae6a45ed1a8 Mon Sep 17 00:00:00 2001 From: jwangace <121262788+jwangace@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:23:06 -0800 Subject: [PATCH 12/13] Update go/vt/vtctl/reparentutil/util_test.go Co-authored-by: Andrew Mason Signed-off-by: jwangace <121262788+jwangace@users.noreply.github.com> --- go/vt/vtctl/reparentutil/util_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 137f09eec99..7b0d624590f 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1219,7 +1219,7 @@ func Test_getTabletsWithPromotionRules(t *testing.T) { } } -func Test_GetBackupCandidates(t *testing.T) { +func TestGetBackupCandidates(t *testing.T) { var ( primaryTablet = &topo.TabletInfo{ Tablet: &topodatapb.Tablet{ From e2c9a0eedab27e4f7f1d93f63a6a8bea9245e1b8 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Wed, 29 Nov 2023 08:58:31 -0800 Subject: [PATCH 13/13] update func GetBackupCandidates Signed-off-by: Jun Wang --- go/vt/vtctl/reparentutil/util.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 2f6f846e793..ac57e1230d1 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -347,14 +347,9 @@ func waitForCatchUp( // GetBackupCandidates is used to get a list of healthy tablets for backup func GetBackupCandidates(tablets []*topo.TabletInfo, stats []*replicationdatapb.Status) (res []*topo.TabletInfo) { for i, stat := range stats { - // Always include TabletType_PRIMARY - if tablets[i].Type == topodatapb.TabletType_PRIMARY { - res = append(res, tablets[i]) - continue - } // shardTablets[i] and stats[i] is 1:1 mapping - // Healthy shardTablets[i] will be added to tablets - if stat != nil { + // Always include TabletType_PRIMARY. Healthy shardTablets[i] will be added to tablets + if tablets[i].Type == topodatapb.TabletType_PRIMARY || stat != nil { res = append(res, tablets[i]) } }