diff --git a/changelog/17.0/17.0.0/summary.md b/changelog/17.0/17.0.0/summary.md new file mode 100644 index 00000000000..27bee34fdb2 --- /dev/null +++ b/changelog/17.0/17.0.0/summary.md @@ -0,0 +1,383 @@ +## Summary + +### Table of Contents + +- **[Major Changes](#major-changes)** + - **[Breaking Changes](#breaking-changes)** + - [Default Local Cell Preference for TabletPicker](#tablet-picker-cell-preference) + - [Dedicated stats for VTGate Prepare operations](#dedicated-vtgate-prepare-stats) + - [VTAdmin web migrated from create-react-app to vite](#migrated-vtadmin) + - [Keyspace name validation in TopoServer](#keyspace-name-validation) + - [Shard name validation in TopoServer](#shard-name-validation) + - [VtctldClient command RestoreFromBackup will now use the correct context](#VtctldClient-RestoreFromBackup) + - **[New command line flags and behavior](#new-flag)** + - [Builtin backup: read buffering flags](#builtin-backup-read-buffering-flags) + - **[New stats](#new-stats)** + - [Detailed backup and restore stats](#detailed-backup-and-restore-stats) + - [VTtablet Error count with code](#vttablet-error-count-with-code) + - [VReplication stream status for Prometheus](#vreplication-stream-status-for-prometheus) + - **[Deprecations and Deletions](#deprecations-and-deletions)** + - [Deprecated Flags](#deprecated-flags) + - [Deprecated Stats](#deprecated-stats) + - **[Vtctld](#vtctld)** + - [Deprecated Flags](#vtctld-deprecated-flags) + - **[VReplication](#VReplication)** + - [Support for MySQL 8.0 `binlog_transaction_compression`](#binlog-compression) + - **[VTTablet](#vttablet)** + - [VTTablet: Initializing all replicas with super_read_only](#vttablet-initialization) + - [Deprecated Flags](#vttablet-deprecated-flags) + - **[VReplication](#VReplication)** + - [Support for the `noblob` binlog row image mode](#noblob) + +## Major Changes + +### Breaking Changes + +#### Default Local Cell Preference for TabletPicker + +We added options to the `TabletPicker` that allow for specifying a cell preference in addition to making the default behavior to give priority to the local cell *and any alias it belongs to*. We are also introducing a new way to select tablet type preference which should eventually replace the `in_order:` hint currently used as a prefix for tablet types. The signature for creating a new `TabletPicker` now looks like: + +``` +func NewTabletPicker( + ctx context.Context, + ts *topo.Server, + cells []string, + localCell, keyspace, shard, tabletTypesStr string, + options TabletPickerOptions, +) (*TabletPicker, error) {...} +``` + +Where ctx, localCell, option are all new parameters. + +`option` is of type `TabletPickerOptions` and includes two fields, `CellPreference` and `TabletOrder`. +CellPreference`: "PreferLocalWithAlias" (default) gives preference to vtgate's local cell, or "OnlySpecified" which only picks from the cells explicitly passed in by the client +`TabletOrder`: "Any" (default) for no ordering or random, or "InOrder" to use the order specified by the client + +See [PR 12282 Description](https://github.com/vitessio/vitess/pull/12282) for examples on how this changes cell picking behavior. + +#### Default TLS version changed for `vtgr` + +When using TLS with `vtgr`, we now default to TLS 1.2 if no other explicit version is configured. Configuration flags are provided to explicitly configure the minimum TLS version to be used. + +#### Dedicated stats for VTGate Prepare operations + +Prior to v17 Vitess incorrectly combined stats for VTGate Execute and Prepare operations under a single stats key (`Execute`). In v17 Execute and Prepare operations generate stats under independent stats keys. + +Here is a (condensed) example of stats output: + +``` +{ + "VtgateApi": { + "Histograms": { + "Execute.src.primary": { + "500000": 5 + }, + "Prepare.src.primary": { + "100000000": 0 + } + } + }, + "VtgateApiErrorCounts": { + "Execute.src.primary.INVALID_ARGUMENT": 3, + "Execute.src.primary.ALREADY_EXISTS": 1 + } +} +``` + +#### VTAdmin web migrated to vite +Previously, VTAdmin web used the Create React App framework to test, build, and serve the application. In v17, Create React App has been removed, and [Vite](https://vitejs.dev/) is used in its place. Some of the main changes include: +- Vite uses `VITE_*` environment variables instead of `REACT_APP_*` environment variables +- Vite uses `import.meta.env` in place of `process.env` +- [Vitest](https://vitest.dev/) is used in place of Jest for testing +- Our protobufjs generator now produces an es6 module instead of commonjs to better work with Vite's defaults +- `public/index.html` has been moved to root directory in web/vtadmin + +#### Keyspace name validation in TopoServer + +Prior to v17, it was possible to create a keyspace with invalid characters, which would then be inaccessible to various cluster management operations. + +Keyspace names are restricted to using only ASCII characters, digits and `_` and `-`. TopoServer's `GetKeyspace` and `CreateKeyspace` methods return an error if given an invalid name. + +#### Shard name validation in TopoServer + +Prior to v17, it was possible to create a shard name with invalid characters, which would then be inaccessible to various cluster management operations. + +Shard names are restricted to using only ASCII characters, digits and `_` and `-`. TopoServer's `GetShard` and `CreateShard` methods return an error if given an invalid name. + +#### VtctldClient command RestoreFromBackup will now use the correct context + +The VtctldClient command RestoreFromBackup initiates an asynchronous process on the specified tablet to restore data from either the latest backup or the closest one before the specified backup-timestamp. +Prior to v17, this asynchronous process could run indefinitely in the background since it was called using the background context. In v17 [PR#12830](https://github.com/vitessio/vitess/issues/12830), +this behavior was changed to use a context with a timeout of `action_timeout`. If you are using VtctldClient to initiate a restore, make sure you provide an appropriate value for action_timeout to give enough +time for the restore process to complete. Otherwise, the restore will throw an error if the context expires before it completes. + +### Vttablet's transaction throttler now also throttles DML outside of `BEGIN; ...; COMMIT;` blocks +Prior to v17, `vttablet`'s transaction throttler (enabled with `--enable-tx-throttler`) would only throttle requests done inside an explicit transaction, i.e., a `BEGIN; ...; COMMIT;` block. +In v17 [PR#13040](https://github.com/vitessio/vitess/issues/13037), this behavior was being changed so that it also throttles work outside of explicit transactions for `INSERT/UPDATE/DELETE/LOAD` queries. + +### New command line flags and behavior + +#### Backup --builtinbackup-file-read-buffer-size and --builtinbackup-file-write-buffer-size + +Prior to v17 the builtin Backup Engine does not use read buffering for restores, and for backups uses a hardcoded write buffer size of 2097152 bytes. + +In v17 these defaults may be tuned with, respectively `--builtinbackup-file-read-buffer-size` and `--builtinbackup-file-write-buffer-size`. + +- `--builtinbackup-file-read-buffer-size`: read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. +- `--builtinbackup-file-write-buffer-size`: write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + +These flags are applicable to the following programs: + +- `vtbackup` +- `vtctld` +- `vttablet` +- `vttestserver` + +### New stats + +#### Detailed backup and restore stats + +##### Backup metrics + +Metrics related to backup operations are available in both Vtbackup and VTTablet. + +**BackupBytes, BackupCount, BackupDurationNanoseconds** + +Depending on the Backup Engine and Backup Storage in-use, a backup may be a complex pipeline of operations, including but not limited to: + +* Reading files from disk. +* Compressing files. +* Uploading compress files to cloud object storage. + +These operations are counted and timed, and the number of bytes consumed or produced by each stage of the pipeline are counted as well. + +##### Restore metrics + +Metrics related to restore operations are available in both Vtbackup and VTTablet. + +**RestoreBytes, RestoreCount, RestoreDurationNanoseconds** + +Depending on the Backup Engine and Backup Storage in-use, a restore may be a complex pipeline of operations, including but not limited to: + +* Downloading compressed files from cloud object storage. +* Decompressing files. +* Writing decompressed files to disk. + +These operations are counted and timed, and the number of bytes consumed or produced by each stage of the pipeline are counted as well. + +##### Vtbackup metrics + +Vtbackup exports some metrics which are not available elsewhere. + +**DurationByPhaseSeconds** + +Vtbackup fetches the last backup, restores it to an empty mysql installation, replicates recent changes into that installation, and then takes a backup of that installation. + +_DurationByPhaseSeconds_ exports timings for these individual phases. + +##### Example + +**A snippet of vtbackup metrics after running it against the local example after creating the initial cluster** + +(Processed with `jq` for readability.) + +``` +{ + "BackupBytes": { + "BackupEngine.Builtin.Source:Read": 4777, + "BackupEngine.Builtin.Compressor:Write": 4616, + "BackupEngine.Builtin.Destination:Write": 162, + "BackupStorage.File.File:Write": 163 + }, + "BackupCount": { + "-.-.Backup": 1, + "BackupEngine.Builtin.Source:Open": 161, + "BackupEngine.Builtin.Source:Close": 322, + "BackupEngine.Builtin.Compressor:Close": 161, + "BackupEngine.Builtin.Destination:Open": 161, + "BackupEngine.Builtin.Destination:Close": 322 + }, + "BackupDurationNanoseconds": { + "-.-.Backup": 4188508542, + "BackupEngine.Builtin.Source:Open": 10649832, + "BackupEngine.Builtin.Source:Read": 55901067, + "BackupEngine.Builtin.Source:Close": 960826, + "BackupEngine.Builtin.Compressor:Write": 278358826, + "BackupEngine.Builtin.Compressor:Close": 79358372, + "BackupEngine.Builtin.Destination:Open": 16456627, + "BackupEngine.Builtin.Destination:Write": 11021043, + "BackupEngine.Builtin.Destination:Close": 17144630, + "BackupStorage.File.File:Write": 10743169 + }, + "DurationByPhaseSeconds": { + "InitMySQLd": 2, + "RestoreLastBackup": 6, + "CatchUpReplication": 1, + "TakeNewBackup": 4 + }, + "RestoreBytes": { + "BackupEngine.Builtin.Source:Read": 1095, + "BackupEngine.Builtin.Decompressor:Read": 950, + "BackupEngine.Builtin.Destination:Write": 209, + "BackupStorage.File.File:Read": 1113 + }, + "RestoreCount": { + "-.-.Restore": 1, + "BackupEngine.Builtin.Source:Open": 161, + "BackupEngine.Builtin.Source:Close": 322, + "BackupEngine.Builtin.Decompressor:Close": 161, + "BackupEngine.Builtin.Destination:Open": 161, + "BackupEngine.Builtin.Destination:Close": 322 + }, + "RestoreDurationNanoseconds": { + "-.-.Restore": 6204765541, + "BackupEngine.Builtin.Source:Open": 10542539, + "BackupEngine.Builtin.Source:Read": 104658370, + "BackupEngine.Builtin.Source:Close": 773038, + "BackupEngine.Builtin.Decompressor:Read": 165692120, + "BackupEngine.Builtin.Decompressor:Close": 51040, + "BackupEngine.Builtin.Destination:Open": 22715122, + "BackupEngine.Builtin.Destination:Write": 41679581, + "BackupEngine.Builtin.Destination:Close": 26954624, + "BackupStorage.File.File:Read": 102416075 + }, + "backup_duration_seconds": 4, + "restore_duration_seconds": 6 +} +``` + +Some notes to help understand these metrics: + +* `BackupBytes["BackupStorage.File.File:Write"]` measures how many bytes were read from disk by the `file` Backup Storage implementation during the backup phase. +* `DurationByPhaseSeconds["CatchUpReplication"]` measures how long it took to catch-up replication after the restore phase. +* `DurationByPhaseSeconds["RestoreLastBackup"]` measures to the duration of the restore phase. +* `RestoreDurationNanoseconds["-.-.Restore"]` also measures to the duration of the restore phase. + +#### VTTablet error count with error code + +##### VTTablet Error Count + +We are introducing new error counter `QueryErrorCountsWithCode` for VTTablet. It is similar to existing [QueryErrorCounts](https://github.com/vitessio/vitess/blob/main/go/vt/vttablet/tabletserver/query_engine.go#L174) except it contains errorCode as additional dimension. +We will deprecate `QueryErrorCounts` in v18. + +#### VReplication stream status for Prometheus + +VReplication publishes the `VReplicationStreamState` status which reports the state of VReplication streams. For example, here's what it looks like in the local cluster example after the MoveTables step: + +``` +"VReplicationStreamState": { + "commerce2customer.1": "Running" +} +``` + +Prior to v17, this data was not available via the Prometheus backend. In v17, workflow states are also published as a Prometheus gauge with a `state` label and a value of `1.0`. For example: + +``` +# HELP vttablet_v_replication_stream_state State of vreplication workflow +# TYPE vttablet_v_replication_stream_state gauge +vttablet_v_replication_stream_state{counts="1",state="Running",workflow="commerce2customer"} 1 +``` + +## Deprecations and Deletions + +* The deprecated `automation` and `automationservice` protobuf definitions and associated client and server packages have been removed. +* Auto-population of DDL revert actions and tables at execution-time has been removed. This is now handled entirely at enqueue-time. +* Backwards-compatibility for failed migrations without a `completed_timestamp` has been removed (see https://github.com/vitessio/vitess/issues/8499). +* The deprecated `Key`, `Name`, `Up`, and `TabletExternallyReparentedTimestamp` fields were removed from the JSON representation of `TabletHealth` structures. + +### Deprecated Command Line Flags + +* Flag `vtctld_addr` has been deprecated and will be deleted in a future release. This affects the `vtgate`, `vttablet` and `vtcombo` binaries. + +### Deprecated Stats + +These stats are deprecated in v17. + +| Deprecated stat | Supported alternatives | +|-|-| +| `backup_duration_seconds` | `BackupDurationNanoseconds` | +| `restore_duration_seconds` | `RestoreDurationNanoseconds` | + +### Vtctld + +#### Deprecated Flags + +The flag `schema_change_check_interval` used to accept either a Go duration value (e.g. `1m` or `30s`) or a bare integer, which was treated as seconds. +This behavior was deprecated in v15.0.0 and has been removed. +`schema_change_check_interval` now **only** accepts Go duration values. + +The flag `durability_policy` is no longer used by vtctld. Instead it reads the durability policies for all keyspaces from the topology server. + +### VTTablet +#### Initializing all replicas with super_read_only +In order to prevent SUPER privileged users like `root` or `vt_dba` from producing errant GTIDs on replicas, all the replica MySQL servers are initialized with the MySQL +global variable `super_read_only` value set to `ON`. During failovers, we set `super_read_only` to `OFF` for the promoted primary tablet. This will allow the +primary to accept writes. All of the shard's tablets, except the current primary, will still have their global variable `super_read_only` set to `ON`. This will make sure that apart from +MySQL replication no other component, offline system or operator can write directly to a replica. + +Reference PR for this change is [PR #12206](https://github.com/vitessio/vitess/pull/12206) + +An important note regarding this change is how the default `init_db.sql` file has changed. +This is even more important if you are running Vitess on the vitess-operator. +You must ensure your `init_db.sql` is up-to-date with the new default for `v17.0.0`. +The default file can be found in `./config/init_db.sql`. + +#### Deprecated Flags +The flag `use_super_read_only` is deprecated and will be removed in a later release. + +Various flags that took float values as seconds have updated to take the standard duration syntax as well. +Float-style parsing is now deprecated and will be removed in a later release. +For example, instead of `--queryserver-config-query-pool-timeout 12.2`, use `--queryserver-config-query-pool-timeout 12s200ms`. +Affected flags and YAML config keys: +- `degraded_threshold` +- `heartbeat_interval` +- `heartbeat_on_demand_duration` +- `health_check_interval` +- `queryserver-config-idle-timeout` +- `queryserver-config-pool-conn-max-lifetime` +- `queryserver-config-olap-transaction-timeout` +- `queryserver-config-query-timeout` +- `queryserver-config-query-pool-timeout` +- `queryserver-config-schema-reload-time` +- `queryserver-config-schema-change-signal-interval` +- `queryserver-config-stream-pool-timeout` +- `queryserver-config-stream-pool-idle-timeout` +- `queryserver-config-transaction-timeout` +- `queryserver-config-txpool-timeout` +- `queryserver-config-txpool-idle-timeout` +- `shutdown_grace_period` +- `unhealthy_threshold` + +### Online DDL + +#### --cut-over-threshold DDL strategy flag + +Online DDL's strategy now accepts `--cut-over-threshold` (type: `duration`) flag. + +This flag stand for the timeout in a `vitess` migration's cut-over phase, which includes the final locking of tables before finalizing the migration. + +The value of the cut-over threshold should be high enough to support the async nature of vreplication catchup phase, as well as accommodate some replication lag. But it mustn't be too high. While cutting over, the migrated table is being locked, causing app connection and query pileup, consuming query buffers, and holding internal mutexes. + +Recommended range for this variable is `5s` - `30s`. Default: `10s`. + +### VReplication + +#### Support for the `noblob` binlog row image mode +The `noblob` binlog row image is now supported by the MoveTables and Reshard VReplication workflows. If the source +or target database has this mode, other workflows like OnlineDDL, Materialize and CreateLookupVindex will error out. +The row events streamed by the VStream API, where blobs and text columns have not changed, will contain null values +for those columns, indicated by a `length:-1`. + +Reference PR for this change is [PR #12905](https://github.com/vitessio/vitess/pull/12905) + +#### Support for MySQL 8.0 binary log transaction compression +MySQL 8.0 added support for [binary log compression via transaction (GTID) compression in 8.0.20](https://dev.mysql.com/blog-archive/mysql-8-0-20-replication-enhancements/). +You can read more about this feature here: https://dev.mysql.com/doc/refman/8.0/en/binary-log-transaction-compression.html + +This can — at the cost of increased CPU usage — dramatically reduce the amount of data sent over the wire for MySQL replication while also dramatically reducing the overall +storage space needed to retain binary logs (for replication, backup and recovery, CDC, etc). For larger installations this was a very desirable feature and while you could +technically use it with Vitess (the MySQL replica-sets making up each shard could use it fine) there was one very big limitation — [VReplication workflows](https://vitess.io/docs/reference/vreplication/vreplication/) +would not work. Given the criticality of VReplication workflows within Vitess, this meant that in practice this MySQL feature was not usable within Vitess clusters. + +We have addressed this issue in [PR #12950](https://github.com/vitessio/vitess/pull/12950) by adding support for processing the compressed transaction events in VReplication, +without any (known) limitations. diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 34f557fc0b5..9c27d0e9c7b 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -212,6 +212,7 @@ func testVStreamCellFlag(t *testing.T) { flags := &vtgatepb.VStreamFlags{} if tc.cells != "" { flags.Cells = tc.cells + flags.CellPreference = "onlyspecified" } ctx2, cancel := context.WithTimeout(ctx, 30*time.Second) diff --git a/go/vt/discovery/tablet_picker.go b/go/vt/discovery/tablet_picker.go index 8f06d60d172..dc52b8d2dcf 100644 --- a/go/vt/discovery/tablet_picker.go +++ b/go/vt/discovery/tablet_picker.go @@ -41,11 +41,40 @@ import ( "vitess.io/vitess/go/vt/vterrors" ) +type TabletPickerCellPreference int + +const ( + // PreferLocalWithAlias gives preference to the local cell first, then specified cells, if any. + // This is the default when no other option is provided. + TabletPickerCellPreference_PreferLocalWithAlias TabletPickerCellPreference = iota + // OnlySpecified only picks tablets from the list of cells given. + TabletPickerCellPreference_OnlySpecified +) + +type TabletPickerTabletOrder int + +const ( + // All provided tablet types are given equal priority. This is the default. + TabletPickerTabletOrder_Any TabletPickerTabletOrder = iota + // Provided tablet types are expected to be prioritized in the given order. + TabletPickerTabletOrder_InOrder +) + var ( tabletPickerRetryDelay = 30 * time.Second muTabletPickerRetryDelay sync.Mutex globalTPStats *tabletPickerStats inOrderHint = "in_order:" + + tabletPickerCellPreferenceMap = map[string]TabletPickerCellPreference{ + "preferlocalwithalias": TabletPickerCellPreference_PreferLocalWithAlias, + "onlyspecified": TabletPickerCellPreference_OnlySpecified, + } + + tabletPickerTabletOrderMap = map[string]TabletPickerTabletOrder{ + "any": TabletPickerTabletOrder_Any, + "inorder": TabletPickerTabletOrder_InOrder, + } ) // GetTabletPickerRetryDelay synchronizes changes to tabletPickerRetryDelay. Used in tests only at the moment @@ -62,18 +91,63 @@ func SetTabletPickerRetryDelay(delay time.Duration) { tabletPickerRetryDelay = delay } +type TabletPickerOptions struct { + CellPreference string + TabletOrder string +} + +func parseTabletPickerCellPreferenceString(str string) (TabletPickerCellPreference, error) { + // return default if blank + if str == "" { + return TabletPickerCellPreference_PreferLocalWithAlias, nil + } + + if c, ok := tabletPickerCellPreferenceMap[strings.ToLower(str)]; ok { + return c, nil + } + + return -1, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "invalid cell preference: %v", str) +} + +func parseTabletPickerTabletOrderString(str string) (TabletPickerTabletOrder, error) { + // return default if blank + if str == "" { + return TabletPickerTabletOrder_Any, nil + } + + if o, ok := tabletPickerTabletOrderMap[strings.ToLower(str)]; ok { + return o, nil + } + + return -1, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "invalid tablet order type: %v", str) +} + +type localCellInfo struct { + localCell string + cellsInAlias map[string]string +} + // TabletPicker gives a simplified API for picking tablets. type TabletPicker struct { - ts *topo.Server - cells []string - keyspace string - shard string - tabletTypes []topodatapb.TabletType - inOrder bool + ts *topo.Server + cells []string + keyspace string + shard string + tabletTypes []topodatapb.TabletType + inOrder bool + cellPref TabletPickerCellPreference + localCellInfo localCellInfo } // NewTabletPicker returns a TabletPicker. -func NewTabletPicker(ts *topo.Server, cells []string, keyspace, shard, tabletTypesStr string) (*TabletPicker, error) { +func NewTabletPicker( + ctx context.Context, + ts *topo.Server, + cells []string, + localCell, keyspace, shard, tabletTypesStr string, + options TabletPickerOptions, +) (*TabletPicker, error) { + // Keep inOrder parsing here for backward compatability until TabletPickerTabletOrder is fully adopted. tabletTypes, inOrder, err := ParseTabletTypesAndOrder(tabletTypesStr) if err != nil { return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "failed to parse list of tablet types: %v", tabletTypesStr) @@ -92,19 +166,123 @@ func NewTabletPicker(ts *topo.Server, cells []string, keyspace, shard, tabletTyp return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, fmt.Sprintf("Missing required field(s) for tablet picker: %s", strings.Join(missingFields, ", "))) } + + // Resolve tablet picker options + cellPref, err := parseTabletPickerCellPreferenceString(options.CellPreference) + if err != nil { + return nil, err + } + + // For backward compatibility only parse the options for tablet ordering + // if the in_order hint wasn't already specified. Otherwise it could be overridden. + // We can remove this check once the in_order hint is deprecated. + if !inOrder { + order, err := parseTabletPickerTabletOrderString(options.TabletOrder) + if err != nil { + return nil, err + } + switch order { + case TabletPickerTabletOrder_Any: + inOrder = false + case TabletPickerTabletOrder_InOrder: + inOrder = true + } + } + + aliasCellMap := make(map[string]string) + if cellPref == TabletPickerCellPreference_PreferLocalWithAlias { + if localCell == "" { + return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cannot have local cell preference without local cell") + } + + // Add local cell to the list of cells for tablet picking. + // This will be de-duped later if the local cell already exists in the original list - see: dedupeCells() + cells = append(cells, localCell) + aliasName := topo.GetAliasByCell(ctx, ts, localCell) + + // If an alias exists + if aliasName != localCell { + alias, err := ts.GetCellsAlias(ctx, aliasName, false) + if err != nil { + return nil, vterrors.Wrap(err, "error fetching local cell alias") + } + + // Add the aliasName to the list of cells for tablet picking. + cells = append(cells, aliasName) + + // Create a map of the cells in the alias to make lookup faster later when we're giving preference to these. + // see prioritizeTablets() + for _, c := range alias.Cells { + aliasCellMap[c] = c + } + } + } + return &TabletPicker{ - ts: ts, - cells: cells, - keyspace: keyspace, - shard: shard, - tabletTypes: tabletTypes, - inOrder: inOrder, + ts: ts, + cells: dedupeCells(cells), + localCellInfo: localCellInfo{localCell: localCell, cellsInAlias: aliasCellMap}, + keyspace: keyspace, + shard: shard, + tabletTypes: tabletTypes, + inOrder: inOrder, + cellPref: cellPref, }, nil } -// PickForStreaming picks an available tablet -// All tablets that belong to tp.cells are evaluated and one is -// chosen at random +// dedupeCells is used to remove duplicates in the cell list in case it is passed in +// and exists in the local cell's alias. Can happen if CellPreference is PreferLocalWithAlias. +func dedupeCells(cells []string) []string { + keys := make(map[string]bool) + dedupedCells := []string{} + + for _, c := range cells { + if _, value := keys[c]; !value { + keys[c] = true + dedupedCells = append(dedupedCells, c) + } + } + return dedupedCells +} + +// prioritizeTablets orders the candidate pool of tablets based on CellPreference. +// If CellPreference is PreferLocalWithAlias then tablets in the local cell will be prioritized for selection, +// followed by the tablets within the local cell's alias, and finally any others specified by the client. +// If CellPreference is OnlySpecified, then tablets will only be selected randomly from the cells specified by the client. +func (tp *TabletPicker) prioritizeTablets(candidates []*topo.TabletInfo) (sameCell, sameAlias, allOthers []*topo.TabletInfo) { + for _, c := range candidates { + if c.Alias.Cell == tp.localCellInfo.localCell { + sameCell = append(sameCell, c) + } else if _, ok := tp.localCellInfo.cellsInAlias[c.Alias.Cell]; ok { + sameAlias = append(sameAlias, c) + } else { + allOthers = append(allOthers, c) + } + } + + return sameCell, sameAlias, allOthers +} + +func (tp *TabletPicker) orderByTabletType(candidates []*topo.TabletInfo) []*topo.TabletInfo { + // Sort candidates slice such that tablets appear in same tablet type order as in tp.tabletTypes + orderMap := map[topodatapb.TabletType]int{} + for i, t := range tp.tabletTypes { + orderMap[t] = i + } + sort.Slice(candidates, func(i, j int) bool { + if orderMap[candidates[i].Type] == orderMap[candidates[j].Type] { + // identical tablet types: randomize order of tablets for this type + return rand.Intn(2) == 0 // 50% chance + } + return orderMap[candidates[i].Type] < orderMap[candidates[j].Type] + }) + + return candidates +} + +// PickForStreaming picks an available tablet. +// Selection is based on CellPreference. +// See prioritizeTablets for prioritization logic. func (tp *TabletPicker) PickForStreaming(ctx context.Context) (*topodatapb.Tablet, error) { rand.Seed(time.Now().UnixNano()) // keep trying at intervals (tabletPickerRetryDelay) until a tablet is found @@ -116,19 +294,30 @@ func (tp *TabletPicker) PickForStreaming(ctx context.Context) (*topodatapb.Table default: } candidates := tp.GetMatchingTablets(ctx) - if tp.inOrder { - // Sort candidates slice such that tablets appear in same tablet type order as in tp.tabletTypes - orderMap := map[topodatapb.TabletType]int{} - for i, t := range tp.tabletTypes { - orderMap[t] = i + if tp.cellPref == TabletPickerCellPreference_PreferLocalWithAlias { + sameCellCandidates, sameAliasCandidates, allOtherCandidates := tp.prioritizeTablets(candidates) + + if tp.inOrder { + sameCellCandidates = tp.orderByTabletType(sameCellCandidates) + sameAliasCandidates = tp.orderByTabletType(sameAliasCandidates) + allOtherCandidates = tp.orderByTabletType(allOtherCandidates) + } else { + // Randomize candidates + rand.Shuffle(len(sameCellCandidates), func(i, j int) { + sameCellCandidates[i], sameCellCandidates[j] = sameCellCandidates[j], sameCellCandidates[i] + }) + rand.Shuffle(len(sameAliasCandidates), func(i, j int) { + sameAliasCandidates[i], sameAliasCandidates[j] = sameAliasCandidates[j], sameAliasCandidates[i] + }) + rand.Shuffle(len(allOtherCandidates), func(i, j int) { + allOtherCandidates[i], allOtherCandidates[j] = allOtherCandidates[j], allOtherCandidates[i] + }) } - sort.Slice(candidates, func(i, j int) bool { - if orderMap[candidates[i].Type] == orderMap[candidates[j].Type] { - // identical tablet types: randomize order of tablets for this type - return rand.Intn(2) == 0 // 50% chance - } - return orderMap[candidates[i].Type] < orderMap[candidates[j].Type] - }) + + candidates = append(sameCellCandidates, sameAliasCandidates...) + candidates = append(candidates, allOtherCandidates...) + } else if tp.inOrder { + candidates = tp.orderByTabletType(candidates) } else { // Randomize candidates rand.Shuffle(len(candidates), func(i, j int) { @@ -204,6 +393,7 @@ func (tp *TabletPicker) GetMatchingTablets(ctx context.Context) []*topo.TabletIn actualCells = append(actualCells, cell) } } + for _, cell := range actualCells { shortCtx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() diff --git a/go/vt/discovery/tablet_picker_test.go b/go/vt/discovery/tablet_picker_test.go index 0f2f3e24318..4a2fae7a2c6 100644 --- a/go/vt/discovery/tablet_picker_test.go +++ b/go/vt/discovery/tablet_picker_test.go @@ -1,12 +1,9 @@ /* Copyright 2019 The Vitess Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,170 +28,7 @@ import ( "vitess.io/vitess/go/vt/topo/memorytopo" ) -func TestPickSimple(t *testing.T) { - te := newPickerTestEnv(t, []string{"cell"}) - want := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) - defer deleteTablet(t, te, want) - - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "replica") - require.NoError(t, err) - - tablet, err := tp.PickForStreaming(context.Background()) - require.NoError(t, err) - assert.True(t, proto.Equal(want, tablet), "Pick: %v, want %v", tablet, want) -} - -func TestPickFromTwoHealthy(t *testing.T) { - te := newPickerTestEnv(t, []string{"cell"}) - want1 := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) - defer deleteTablet(t, te, want1) - want2 := addTablet(te, 101, topodatapb.TabletType_RDONLY, "cell", true, true) - defer deleteTablet(t, te, want2) - - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "replica,rdonly") - require.NoError(t, err) - - // In 20 attempts, both tablet types must be picked at least once. - var picked1, picked2 bool - for i := 0; i < 20; i++ { - tablet, err := tp.PickForStreaming(context.Background()) - require.NoError(t, err) - if proto.Equal(tablet, want1) { - picked1 = true - } - if proto.Equal(tablet, want2) { - picked2 = true - } - } - assert.True(t, picked1) - assert.True(t, picked2) -} - -func TestPickInOrder1(t *testing.T) { - te := newPickerTestEnv(t, []string{"cell"}) - want1 := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) - defer deleteTablet(t, te, want1) - want2 := addTablet(te, 101, topodatapb.TabletType_RDONLY, "cell", true, true) - defer deleteTablet(t, te, want2) - - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "in_order:replica,rdonly") - require.NoError(t, err) - - // In 20 attempts, we always pick the first healthy tablet in order - var picked1, picked2 bool - for i := 0; i < 20; i++ { - tablet, err := tp.PickForStreaming(context.Background()) - require.NoError(t, err) - if proto.Equal(tablet, want1) { - picked1 = true - } - if proto.Equal(tablet, want2) { - picked2 = true - } - } - assert.True(t, picked1) - assert.False(t, picked2) -} - -func TestPickInOrder2(t *testing.T) { - te := newPickerTestEnv(t, []string{"cell"}) - want1 := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) - defer deleteTablet(t, te, want1) - want2 := addTablet(te, 101, topodatapb.TabletType_RDONLY, "cell", true, true) - defer deleteTablet(t, te, want2) - - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "in_order:rdonly,replica") - require.NoError(t, err) - - // In 20 attempts, we always pick the first healthy tablet in order - var picked1, picked2 bool - for i := 0; i < 20; i++ { - tablet, err := tp.PickForStreaming(context.Background()) - require.NoError(t, err) - if proto.Equal(tablet, want1) { - picked1 = true - } - if proto.Equal(tablet, want2) { - picked2 = true - } - } - assert.False(t, picked1) - assert.True(t, picked2) -} - -func TestPickInOrderMultipleInGroup(t *testing.T) { - te := newPickerTestEnv(t, []string{"cell"}) - want1 := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) - defer deleteTablet(t, te, want1) - want2 := addTablet(te, 101, topodatapb.TabletType_RDONLY, "cell", true, true) - defer deleteTablet(t, te, want2) - want3 := addTablet(te, 102, topodatapb.TabletType_RDONLY, "cell", true, true) - defer deleteTablet(t, te, want3) - want4 := addTablet(te, 103, topodatapb.TabletType_RDONLY, "cell", true, true) - defer deleteTablet(t, te, want4) - - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "in_order:rdonly,replica") - require.NoError(t, err) - - // In 40 attempts, we pick each of the three RDONLY, but never the REPLICA - var picked1, picked2, picked3, picked4 bool - for i := 0; i < 40; i++ { - tablet, err := tp.PickForStreaming(context.Background()) - require.NoError(t, err) - if proto.Equal(tablet, want1) { - picked1 = true - } - if proto.Equal(tablet, want2) { - picked2 = true - } - if proto.Equal(tablet, want3) { - picked3 = true - } - if proto.Equal(tablet, want4) { - picked4 = true - } - } - assert.False(t, picked1) - assert.True(t, picked2) - assert.True(t, picked3) - assert.True(t, picked4) -} - -func TestPickRespectsTabletType(t *testing.T) { - te := newPickerTestEnv(t, []string{"cell"}) - want := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) - defer deleteTablet(t, te, want) - dont := addTablet(te, 101, topodatapb.TabletType_MASTER, "cell", true, true) - defer deleteTablet(t, te, dont) - - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "replica,rdonly") - require.NoError(t, err) - - // In 20 attempts, master tablet must be never picked - for i := 0; i < 20; i++ { - tablet, err := tp.PickForStreaming(context.Background()) - require.NoError(t, err) - require.NotNil(t, tablet) - require.True(t, proto.Equal(tablet, want), "picked wrong tablet type") - } -} - -func TestPickMultiCell(t *testing.T) { - te := newPickerTestEnv(t, []string{"cell", "otherCell"}) - want := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) - defer deleteTablet(t, te, want) - - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "replica") - require.NoError(t, err) - - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - tablet, err := tp.PickForStreaming(ctx) - require.NoError(t, err) - assert.True(t, proto.Equal(want, tablet), "Pick: %v, want %v", tablet, want) -} - -func TestPickMaster(t *testing.T) { +func TestPickPrimary(t *testing.T) { te := newPickerTestEnv(t, []string{"cell", "otherCell"}) want := addTablet(te, 100, topodatapb.TabletType_MASTER, "cell", true, true) defer deleteTablet(t, te, want) @@ -206,7 +40,7 @@ func TestPickMaster(t *testing.T) { }) require.NoError(t, err) - tp, err := NewTabletPicker(te.topoServ, []string{"otherCell"}, te.keyspace, te.shard, "master") + tp, err := NewTabletPicker(context.Background(), te.topoServ, []string{"otherCell"}, "cell", te.keyspace, te.shard, "master", TabletPickerOptions{}) require.NoError(t, err) ctx2, cancel2 := context.WithTimeout(context.Background(), 200*time.Millisecond) @@ -216,67 +50,278 @@ func TestPickMaster(t *testing.T) { assert.True(t, proto.Equal(want, tablet), "Pick: %v, want %v", tablet, want) } -func TestPickFromOtherCell(t *testing.T) { - te := newPickerTestEnv(t, []string{"cell", "otherCell"}) - want := addTablet(te, 100, topodatapb.TabletType_REPLICA, "otherCell", true, true) - defer deleteTablet(t, te, want) +func TestPickLocalPreferences(t *testing.T) { + type tablet struct { + id uint32 + typ topodatapb.TabletType + cell string + } - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "replica") - require.NoError(t, err) + type testCase struct { + name string - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - tablet, err := tp.PickForStreaming(ctx) - require.NoError(t, err) - assert.True(t, proto.Equal(want, tablet), "Pick: %v, want %v", tablet, want) -} - -func TestDontPickFromOtherCell(t *testing.T) { - te := newPickerTestEnv(t, []string{"cell", "otherCell"}) - want1 := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) - defer deleteTablet(t, te, want1) - want2 := addTablet(te, 101, topodatapb.TabletType_REPLICA, "otherCell", true, true) - defer deleteTablet(t, te, want2) + //inputs + tablets []tablet + envCells []string + inCells []string + localCell string + inTabletTypes string + options TabletPickerOptions - tp, err := NewTabletPicker(te.topoServ, []string{"cell"}, te.keyspace, te.shard, "replica") - require.NoError(t, err) + //expected + tpCells []string + wantTablets []uint32 + } - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() + tcases := []testCase{ + { + name: "pick simple", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + }, + envCells: []string{"cell"}, + inCells: []string{"cell"}, + localCell: "cell", + inTabletTypes: "replica", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "cella"}, + wantTablets: []uint32{100}, + }, { + name: "pick from two healthy", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_RDONLY, "cell"}, + }, + envCells: []string{"cell"}, + inCells: []string{"cell"}, + localCell: "cell", + inTabletTypes: "replica,rdonly", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "cella"}, + wantTablets: []uint32{100, 101}, + }, { + name: "pick in order replica", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_RDONLY, "cell"}, + }, + envCells: []string{"cell"}, + inCells: []string{"cell"}, + localCell: "cell", + inTabletTypes: "in_order:replica,rdonly", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "cella"}, + wantTablets: []uint32{100}, + }, { + name: "pick in order rdonly", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_RDONLY, "cell"}, + }, + envCells: []string{"cell"}, + inCells: []string{"cell"}, + localCell: "cell", + inTabletTypes: "in_order:rdonly,replica", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "cella"}, + wantTablets: []uint32{101}, + }, { + name: "pick in order multiple in group", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_RDONLY, "cell"}, + {102, topodatapb.TabletType_RDONLY, "cell"}, + {103, topodatapb.TabletType_RDONLY, "cell"}, + }, + envCells: []string{"cell"}, + inCells: []string{"cell"}, + localCell: "cell", + inTabletTypes: "in_order:rdonly,replica", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "cella"}, + wantTablets: []uint32{101, 102, 103}, + }, { + // Same test as above, except the in order preference is passed via the new TabletPickerOptions param. + // This will replace the above test when we deprecate the "in_order" hint in the tabletTypeStr + name: "pick in order multiple in group with new picker option", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_RDONLY, "cell"}, + {102, topodatapb.TabletType_RDONLY, "cell"}, + {103, topodatapb.TabletType_RDONLY, "cell"}, + }, + envCells: []string{"cell"}, + inCells: []string{"cell"}, + localCell: "cell", + inTabletTypes: "rdonly,replica", + options: TabletPickerOptions{TabletOrder: "InOrder"}, + tpCells: []string{"cell", "cella"}, + wantTablets: []uint32{101, 102, 103}, + }, { + name: "picker respects tablet type", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_MASTER, "cell"}, + }, + envCells: []string{"cell"}, + inCells: []string{"cell"}, + localCell: "cell", + inTabletTypes: "replica,rdonly", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "cella"}, + wantTablets: []uint32{100}, + }, { + name: "pick multi cell", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + }, + envCells: []string{"cell", "otherCell"}, + inCells: []string{"cell", "otherCell"}, + localCell: "cell", + inTabletTypes: "replica", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "otherCell", "cella"}, + wantTablets: []uint32{100}, + }, { + name: "pick from other cell", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "otherCell"}, + }, + envCells: []string{"cell", "otherCell"}, + inCells: []string{"cell", "otherCell"}, + localCell: "cell", + inTabletTypes: "replica", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "otherCell", "cella"}, + wantTablets: []uint32{100}, + }, { + name: "don't pick from other cell", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_REPLICA, "otherCell"}, + }, + envCells: []string{"cell", "otherCell"}, + inCells: []string{"cell"}, + localCell: "cell", + inTabletTypes: "replica", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "cella"}, + wantTablets: []uint32{100}, + }, { + name: "multi cell two tablets, local preference default", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_REPLICA, "otherCell"}, + }, + envCells: []string{"cell", "otherCell"}, + inCells: []string{"cell", "otherCell"}, + localCell: "cell", + inTabletTypes: "replica", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "otherCell", "cella"}, + wantTablets: []uint32{100}, + }, { + name: "multi cell two tablets, only specified cells", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_REPLICA, "otherCell"}, + }, + envCells: []string{"cell", "otherCell"}, + inCells: []string{"cell", "otherCell"}, + localCell: "cell", + inTabletTypes: "replica", + options: TabletPickerOptions{CellPreference: "OnlySpecified"}, + tpCells: []string{"cell", "otherCell"}, + wantTablets: []uint32{100, 101}, + }, { + name: "multi cell two tablet types, local preference default", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_RDONLY, "otherCell"}, + }, + envCells: []string{"cell", "otherCell"}, + inCells: []string{"cell", "otherCell"}, + localCell: "cell", + inTabletTypes: "replica,rdonly", + options: TabletPickerOptions{}, + tpCells: []string{"cell", "otherCell", "cella"}, + wantTablets: []uint32{100}, + }, { + name: "multi cell two tablet types, only specified cells", + tablets: []tablet{ + {100, topodatapb.TabletType_REPLICA, "cell"}, + {101, topodatapb.TabletType_RDONLY, "otherCell"}, + }, + envCells: []string{"cell", "otherCell"}, + inCells: []string{"cell", "otherCell"}, + localCell: "cell", + inTabletTypes: "replica,rdonly", + options: TabletPickerOptions{CellPreference: "OnlySpecified"}, + tpCells: []string{"cell", "otherCell"}, + wantTablets: []uint32{100, 101}, + }, + } - // In 20 attempts, only want1 must be picked because TabletPicker.cells = "cell" - var picked1, picked2 bool - for i := 0; i < 20; i++ { - tablet, err := tp.PickForStreaming(ctx) - require.NoError(t, err) - if proto.Equal(tablet, want1) { - picked1 = true - } - if proto.Equal(tablet, want2) { - picked2 = true - } + ctx := context.Background() + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + te := newPickerTestEnv(t, tcase.envCells) + var testTablets []*topodatapb.Tablet + for _, tab := range tcase.tablets { + testTablets = append(testTablets, addTablet(te, int(tab.id), tab.typ, tab.cell, true, true)) + } + defer func() { + for _, tab := range testTablets { + deleteTablet(t, te, tab) + } + }() + tp, err := NewTabletPicker(context.Background(), te.topoServ, tcase.inCells, tcase.localCell, te.keyspace, te.shard, tcase.inTabletTypes, tcase.options) + require.NoError(t, err) + require.Equal(t, tp.localCellInfo.localCell, tcase.localCell) + require.ElementsMatch(t, tp.cells, tcase.tpCells) + + var selectedTablets []uint32 + selectedTabletMap := make(map[uint32]bool) + for i := 0; i < 40; i++ { + tab, err := tp.PickForStreaming(ctx) + require.NoError(t, err) + selectedTabletMap[tab.Alias.Uid] = true + } + for uid := range selectedTabletMap { + selectedTablets = append(selectedTablets, uid) + } + require.ElementsMatch(t, selectedTablets, tcase.wantTablets) + }) } - assert.True(t, picked1) - assert.False(t, picked2) } -func TestPickMultiCellTwoTablets(t *testing.T) { +func TestPickCellPreferenceLocalCell(t *testing.T) { + // test env puts all cells into an alias called "cella" te := newPickerTestEnv(t, []string{"cell", "otherCell"}) want1 := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) defer deleteTablet(t, te, want1) - want2 := addTablet(te, 101, topodatapb.TabletType_REPLICA, "otherCell", true, true) - defer deleteTablet(t, te, want2) - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "replica") + // Local cell preference is default + tp, err := NewTabletPicker(context.Background(), te.topoServ, []string{"cella"}, "cell", te.keyspace, te.shard, "replica", TabletPickerOptions{}) require.NoError(t, err) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() + ctx1, cancel1 := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel1() + tablet, err := tp.PickForStreaming(ctx1) + require.NoError(t, err) + assert.True(t, proto.Equal(want1, tablet), "Pick: %v, want %v", tablet, want1) + + // create a tablet in the other cell + want2 := addTablet(te, 101, topodatapb.TabletType_REPLICA, "otherCell", true, true) + defer deleteTablet(t, te, want2) + + ctx2, cancel2 := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel2() - // In 20 attempts, both tablet types must be picked at least once. + // In 20 attempts, only tablet in "cell" will be picked because we give local cell priority by default var picked1, picked2 bool for i := 0; i < 20; i++ { - tablet, err := tp.PickForStreaming(ctx) + tablet, err := tp.PickForStreaming(ctx2) require.NoError(t, err) if proto.Equal(tablet, want1) { picked1 = true @@ -286,45 +331,32 @@ func TestPickMultiCellTwoTablets(t *testing.T) { } } assert.True(t, picked1) - assert.True(t, picked2) + assert.False(t, picked2) } -func TestPickMultiCellTwoTabletTypes(t *testing.T) { +func TestPickCellPreferenceLocalAlias(t *testing.T) { + // test env puts all cells into an alias called "cella" te := newPickerTestEnv(t, []string{"cell", "otherCell"}) - want1 := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) - defer deleteTablet(t, te, want1) - want2 := addTablet(te, 101, topodatapb.TabletType_RDONLY, "otherCell", true, true) - defer deleteTablet(t, te, want2) - - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "replica,rdonly") + tp, err := NewTabletPicker(context.Background(), te.topoServ, []string{"cella"}, "cell", te.keyspace, te.shard, "replica", TabletPickerOptions{}) require.NoError(t, err) + // create a tablet in the other cell, it should be picked + want := addTablet(te, 101, topodatapb.TabletType_REPLICA, "otherCell", true, true) + defer deleteTablet(t, te, want) ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - - // In 20 attempts, both tablet types must be picked at least once. - var picked1, picked2 bool - for i := 0; i < 20; i++ { - tablet, err := tp.PickForStreaming(ctx) - require.NoError(t, err) - if proto.Equal(tablet, want1) { - picked1 = true - } - if proto.Equal(tablet, want2) { - picked2 = true - } - } - assert.True(t, picked1) - assert.True(t, picked2) + tablet, err := tp.PickForStreaming(ctx) + require.NoError(t, err) + assert.True(t, proto.Equal(want, tablet), "Pick: %v, want %v", tablet, want) } -func TestPickUsingCellAlias(t *testing.T) { +func TestPickUsingCellAliasOnlySpecified(t *testing.T) { // test env puts all cells into an alias called "cella" te := newPickerTestEnv(t, []string{"cell", "otherCell"}) want1 := addTablet(te, 100, topodatapb.TabletType_REPLICA, "cell", true, true) defer deleteTablet(t, te, want1) - tp, err := NewTabletPicker(te.topoServ, []string{"cella"}, te.keyspace, te.shard, "replica") + tp, err := NewTabletPicker(context.Background(), te.topoServ, []string{"cella"}, "cell", te.keyspace, te.shard, "replica", TabletPickerOptions{CellPreference: "OnlySpecified"}) require.NoError(t, err) ctx1, cancel1 := context.WithTimeout(context.Background(), 200*time.Millisecond) @@ -348,7 +380,8 @@ func TestPickUsingCellAlias(t *testing.T) { ctx3, cancel3 := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel3() - // In 20 attempts, both tablet types must be picked at least once. + // In 20 attempts each of the tablets should get picked at least once. + // Local cell is not given preference var picked1, picked2 bool for i := 0; i < 20; i++ { tablet, err := tp.PickForStreaming(ctx3) @@ -366,7 +399,7 @@ func TestPickUsingCellAlias(t *testing.T) { func TestTabletAppearsDuringSleep(t *testing.T) { te := newPickerTestEnv(t, []string{"cell"}) - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "replica") + tp, err := NewTabletPicker(context.Background(), te.topoServ, te.cells, "cell", te.keyspace, te.shard, "replica", TabletPickerOptions{}) require.NoError(t, err) delay := GetTabletPickerRetryDelay() @@ -392,12 +425,12 @@ func TestTabletAppearsDuringSleep(t *testing.T) { assert.True(t, proto.Equal(want, got), "Pick: %v, want %v", got, want) } -func TestPickError(t *testing.T) { +func TestPickErrorLocalPreferenceDefault(t *testing.T) { te := newPickerTestEnv(t, []string{"cell"}) - _, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "badtype") + _, err := NewTabletPicker(context.Background(), te.topoServ, te.cells, "cell", te.keyspace, te.shard, "badtype", TabletPickerOptions{}) assert.EqualError(t, err, "failed to parse list of tablet types: badtype") - tp, err := NewTabletPicker(te.topoServ, te.cells, te.keyspace, te.shard, "replica") + tp, err := NewTabletPicker(context.Background(), te.topoServ, te.cells, "cell", te.keyspace, te.shard, "replica", TabletPickerOptions{}) require.NoError(t, err) delay := GetTabletPickerRetryDelay() defer func() { @@ -416,6 +449,33 @@ func TestPickError(t *testing.T) { defer cancel() _, err = tp.PickForStreaming(ctx) require.EqualError(t, err, "context has expired") + // if local preference is selected, tp cells include's the local cell's alias + require.Greater(t, globalTPStats.noTabletFoundError.Counts()["cell_cella.ks.0.replica"], int64(0)) +} + +func TestPickErrorOnlySpecified(t *testing.T) { + te := newPickerTestEnv(t, []string{"cell"}) + + tp, err := NewTabletPicker(context.Background(), te.topoServ, te.cells, "cell", te.keyspace, te.shard, "replica", TabletPickerOptions{CellPreference: "OnlySpecified"}) + require.NoError(t, err) + delay := GetTabletPickerRetryDelay() + defer func() { + SetTabletPickerRetryDelay(delay) + }() + SetTabletPickerRetryDelay(11 * time.Millisecond) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + // no tablets + _, err = tp.PickForStreaming(ctx) + require.EqualError(t, err, "context has expired") + // no tablets of the correct type + defer deleteTablet(t, te, addTablet(te, 200, topodatapb.TabletType_RDONLY, "cell", true, true)) + ctx, cancel = context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + _, err = tp.PickForStreaming(ctx) + require.EqualError(t, err, "context has expired") + require.Greater(t, globalTPStats.noTabletFoundError.Counts()["cell.ks.0.replica"], int64(0)) } diff --git a/go/vt/proto/vtgate/vtgate.pb.go b/go/vt/proto/vtgate/vtgate.pb.go index 31babb3f4d5..792258e7a90 100644 --- a/go/vt/proto/vtgate/vtgate.pb.go +++ b/go/vt/proto/vtgate/vtgate.pb.go @@ -1066,7 +1066,9 @@ type VStreamFlags struct { StopOnReshard bool `protobuf:"varint,3,opt,name=stop_on_reshard,json=stopOnReshard,proto3" json:"stop_on_reshard,omitempty"` // if specified, these cells (comma-separated) are used to pick source tablets from. // defaults to the cell of the vtgate serving the VStream API. - Cells string `protobuf:"bytes,4,opt,name=cells,proto3" json:"cells,omitempty"` + Cells string `protobuf:"bytes,4,opt,name=cells,proto3" json:"cells,omitempty"` + CellPreference string `protobuf:"bytes,5,opt,name=cell_preference,json=cellPreference,proto3" json:"cell_preference,omitempty"` + TabletOrder string `protobuf:"bytes,6,opt,name=tablet_order,json=tabletOrder,proto3" json:"tablet_order,omitempty"` } func (x *VStreamFlags) Reset() { @@ -1129,6 +1131,20 @@ func (x *VStreamFlags) GetCells() string { return "" } +func (x *VStreamFlags) GetCellPreference() string { + if x != nil { + return x.CellPreference + } + return "" +} + +func (x *VStreamFlags) GetTabletOrder() string { + if x != nil { + return x.TabletOrder + } + return "" +} + // VStreamRequest is the payload for VStream. type VStreamRequest struct { state protoimpl.MessageState diff --git a/go/vt/proto/vtgate/vtgate_vtproto.pb.go b/go/vt/proto/vtgate/vtgate_vtproto.pb.go index 03da20424fb..64cf29596e6 100644 --- a/go/vt/proto/vtgate/vtgate_vtproto.pb.go +++ b/go/vt/proto/vtgate/vtgate_vtproto.pb.go @@ -1017,6 +1017,20 @@ func (m *VStreamFlags) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.TabletOrder) > 0 { + i -= len(m.TabletOrder) + copy(dAtA[i:], m.TabletOrder) + i = encodeVarint(dAtA, i, uint64(len(m.TabletOrder))) + i-- + dAtA[i] = 0x32 + } + if len(m.CellPreference) > 0 { + i -= len(m.CellPreference) + copy(dAtA[i:], m.CellPreference) + i = encodeVarint(dAtA, i, uint64(len(m.CellPreference))) + i-- + dAtA[i] = 0x2a + } if len(m.Cells) > 0 { i -= len(m.Cells) copy(dAtA[i:], m.Cells) @@ -4552,6 +4566,70 @@ func (m *VStreamFlags) UnmarshalVT(dAtA []byte) error { } m.Cells = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CellPreference", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CellPreference = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TabletOrder", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TabletOrder = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skip(dAtA[iNdEx:]) diff --git a/go/vt/vtgate/vstream_manager.go b/go/vt/vtgate/vstream_manager.go index a3e53c8d7c3..e27024decc4 100644 --- a/go/vt/vtgate/vstream_manager.go +++ b/go/vt/vtgate/vstream_manager.go @@ -105,6 +105,8 @@ type vstream struct { eventCh chan []*binlogdatapb.VEvent heartbeatInterval uint32 ts *topo.Server + + tabletPickerOptions discovery.TabletPickerOptions } type journalEvent struct { @@ -151,6 +153,10 @@ func (vsm *vstreamManager) VStream(ctx context.Context, tabletType topodatapb.Ta eventCh: make(chan []*binlogdatapb.VEvent), heartbeatInterval: flags.GetHeartbeatInterval(), ts: ts, + tabletPickerOptions: discovery.TabletPickerOptions{ + CellPreference: flags.GetCellPreference(), + TabletOrder: flags.GetTabletOrder(), + }, } return vs.stream(ctx) } @@ -447,7 +453,7 @@ func (vs *vstream) streamFromTablet(ctx context.Context, sgtid *binlogdatapb.Sha var eventss [][]*binlogdatapb.VEvent var err error cells := vs.getCells() - tp, err := discovery.NewTabletPicker(vs.ts, cells, sgtid.Keyspace, sgtid.Shard, vs.tabletType.String()) + tp, err := discovery.NewTabletPicker(ctx, vs.ts, cells, vs.vsm.cell, sgtid.Keyspace, sgtid.Shard, vs.tabletType.String(), vs.tabletPickerOptions) if err != nil { log.Errorf(err.Error()) return err diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 8776a37a131..f676cafaf05 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -127,7 +127,7 @@ func newController(ctx context.Context, params map[string]string, dbClientFactor return nil, err } } - tp, err := discovery.NewTabletPicker(sourceTopo, cells, ct.source.Keyspace, ct.source.Shard, tabletTypesStr) + tp, err := discovery.NewTabletPicker(ctx, sourceTopo, cells, ct.vre.cell, ct.source.Keyspace, ct.source.Shard, tabletTypesStr, discovery.TabletPickerOptions{}) if err != nil { return nil, err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go index a54145b3fb2..220cfe625a5 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go @@ -76,8 +76,9 @@ func TestControllerKeyRange(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} + vre := NewTestEngine(nil, wantTablet.GetAlias().Cell, mysqld, dbClientFactory, dbClientFactory, dbClient.DBName(), nil) - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, vre) if err != nil { t.Fatal(err) } @@ -137,8 +138,9 @@ func TestControllerTables(t *testing.T) { }, }, } + vre := NewTestEngine(nil, wantTablet.GetAlias().Cell, mysqld, dbClientFactory, dbClientFactory, dbClient.DBName(), nil) - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, vre) if err != nil { t.Fatal(err) } @@ -205,8 +207,9 @@ func TestControllerOverrides(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} + vre := NewTestEngine(nil, wantTablet.GetAlias().Cell, mysqld, dbClientFactory, dbClientFactory, dbClient.DBName(), nil) - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, vre) if err != nil { t.Fatal(err) } @@ -220,7 +223,8 @@ func TestControllerOverrides(t *testing.T) { } func TestControllerCanceledContext(t *testing.T) { - defer deleteTablet(addTablet(100)) + wantTablet := addTablet(100) + defer deleteTablet(wantTablet) params := map[string]string{ "id": "1", @@ -230,7 +234,9 @@ func TestControllerCanceledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - ct, err := newController(ctx, params, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil, nil) + vre := NewTestEngine(nil, wantTablet.GetAlias().Cell, nil, nil, nil, "", nil) + + ct, err := newController(ctx, params, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil, vre) if err != nil { t.Fatal(err) } @@ -273,8 +279,9 @@ func TestControllerRetry(t *testing.T) { dbClient.ExpectRequest("commit", nil, nil) dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} + vre := NewTestEngine(nil, env.Cells[0], mysqld, dbClientFactory, dbClientFactory, dbClient.DBName(), nil) - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, vre) if err != nil { t.Fatal(err) } @@ -319,8 +326,9 @@ func TestControllerStopPosition(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} + vre := NewTestEngine(nil, wantTablet.GetAlias().Cell, mysqld, dbClientFactory, dbClientFactory, dbClient.DBName(), nil) - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, vre) if err != nil { t.Fatal(err) } diff --git a/go/vt/wrangler/traffic_switcher.go b/go/vt/wrangler/traffic_switcher.go index 60a5ba9c03a..03b5a46c0a9 100644 --- a/go/vt/wrangler/traffic_switcher.go +++ b/go/vt/wrangler/traffic_switcher.go @@ -448,7 +448,7 @@ func (wr *Wrangler) areTabletsAvailableToStreamFrom(ctx context.Context, ts *tra if cells == nil { cells = append(cells, shard.MasterAlias.Cell) } - tp, err := discovery.NewTabletPicker(wr.ts, cells, keyspace, shard.ShardName(), tabletTypes) + tp, err := discovery.NewTabletPicker(ctx, wr.ts, cells, shard.MasterAlias.Cell, keyspace, shard.ShardName(), tabletTypes, discovery.TabletPickerOptions{}) if err != nil { allErrors.RecordError(err) return diff --git a/go/vt/wrangler/traffic_switcher_env_test.go b/go/vt/wrangler/traffic_switcher_env_test.go index 6194b7c0985..401a8806634 100644 --- a/go/vt/wrangler/traffic_switcher_env_test.go +++ b/go/vt/wrangler/traffic_switcher_env_test.go @@ -392,7 +392,7 @@ func (tme *testMigraterEnv) createDBClients(ctx context.Context, t *testing.T) { tme.dbSourceClients = append(tme.dbSourceClients, dbclient) dbClientFactory := func() binlogplayer.DBClient { return dbclient } // Replace existing engine with a new one - master.TM.VREngine = vreplication.NewTestEngine(tme.ts, "", master.FakeMysqlDaemon, dbClientFactory, dbClientFactory, dbclient.DBName(), nil) + master.TM.VREngine = vreplication.NewTestEngine(tme.ts, master.Tablet.GetAlias().GetCell(), master.FakeMysqlDaemon, dbClientFactory, dbClientFactory, dbclient.DBName(), nil) master.TM.VREngine.Open(ctx) } for _, master := range tme.targetMasters { @@ -401,7 +401,7 @@ func (tme *testMigraterEnv) createDBClients(ctx context.Context, t *testing.T) { tme.dbTargetClients = append(tme.dbTargetClients, dbclient) dbClientFactory := func() binlogplayer.DBClient { return dbclient } // Replace existing engine with a new one - master.TM.VREngine = vreplication.NewTestEngine(tme.ts, "", master.FakeMysqlDaemon, dbClientFactory, dbClientFactory, dbclient.DBName(), nil) + master.TM.VREngine = vreplication.NewTestEngine(tme.ts, master.Tablet.GetAlias().GetCell(), master.FakeMysqlDaemon, dbClientFactory, dbClientFactory, dbclient.DBName(), nil) master.TM.VREngine.Open(ctx) } tme.allDBClients = append(tme.dbSourceClients, tme.dbTargetClients...) diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index 53dade07c1e..f1bcd225d06 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -572,7 +572,7 @@ func (df *vdiff) selectTablets(ctx context.Context, ts *trafficSwitcher) error { if ts.ExternalTopo() != nil { sourceTopo = ts.ExternalTopo() } - tp, err := discovery.NewTabletPicker(sourceTopo, []string{df.sourceCell}, df.ts.SourceKeyspaceName(), shard, df.tabletTypesStr) + tp, err := discovery.NewTabletPicker(ctx, sourceTopo, []string{df.sourceCell}, df.sourceCell, df.ts.SourceKeyspaceName(), shard, df.tabletTypesStr, discovery.TabletPickerOptions{}) if err != nil { return err } @@ -590,7 +590,7 @@ func (df *vdiff) selectTablets(ctx context.Context, ts *trafficSwitcher) error { go func() { defer wg.Done() err2 = df.forAll(df.targets, func(shard string, target *shardStreamer) error { - tp, err := discovery.NewTabletPicker(df.ts.TopoServer(), []string{df.targetCell}, df.ts.TargetKeyspaceName(), shard, df.tabletTypesStr) + tp, err := discovery.NewTabletPicker(ctx, df.ts.TopoServer(), []string{df.targetCell}, df.targetCell, df.ts.TargetKeyspaceName(), shard, df.tabletTypesStr, discovery.TabletPickerOptions{}) if err != nil { return err } diff --git a/proto/vtgate.proto b/proto/vtgate.proto index 923a41ae6d5..e92bf28d46e 100644 --- a/proto/vtgate.proto +++ b/proto/vtgate.proto @@ -280,6 +280,8 @@ message VStreamFlags { // if specified, these cells (comma-separated) are used to pick source tablets from. // defaults to the cell of the vtgate serving the VStream API. string cells = 4; + string cell_preference = 5; + string tablet_order = 6; } // VStreamRequest is the payload for VStream. diff --git a/tools/rowlog/rowlog.go b/tools/rowlog/rowlog.go index 920634dcd23..02fcc9b2525 100644 --- a/tools/rowlog/rowlog.go +++ b/tools/rowlog/rowlog.go @@ -372,7 +372,18 @@ func getFlavor(ctx context.Context, server, keyspace string) string { } func getTablet(ctx context.Context, ts *topo.Server, cells []string, keyspace string) string { - picker, err := discovery.NewTabletPicker(ts, cells, keyspace, "0", "master") + picker, err := discovery.NewTabletPicker( + ctx, + ts, + cells, + "", + keyspace, + "0", + "master", + discovery.TabletPickerOptions{ + CellPreference: "OnlySpecified", + }, + ) if err != nil { return "" }