From c982a0406a292a7a0ff632b79166eade9621f5f8 Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Wed, 21 Feb 2024 14:46:31 +0530 Subject: [PATCH 01/16] Add memory check for runners for VTOrc tests (#15317) Signed-off-by: Manan Gupta --- .github/workflows/cluster_endtoend_vtorc.yml | 9 +++++++++ .../workflows/cluster_endtoend_vtorc_mysql57.yml | 9 +++++++++ test/ci_workflow_gen.go | 11 +++++++++++ test/templates/cluster_endtoend_test.tpl | 13 +++++++++++++ test/templates/cluster_endtoend_test_mysql57.tpl | 13 +++++++++++++ 5 files changed, 55 insertions(+) diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml index 39413d28cde..3dec9b0262d 100644 --- a/.github/workflows/cluster_endtoend_vtorc.yml +++ b/.github/workflows/cluster_endtoend_vtorc.yml @@ -43,6 +43,15 @@ jobs: draft=$(echo "$PR_DATA" | jq .draft -r) echo "is_draft=${draft}" >> $GITHUB_OUTPUT + - name: Check Memory + run: | + totalMem=$(free -g | awk 'NR==2 {print $2}') + echo "total memory $totalMem GB" + if [[ "$totalMem" -lt 15 ]]; then + echo "Less memory than required" + exit 1 + fi + - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 diff --git a/.github/workflows/cluster_endtoend_vtorc_mysql57.yml b/.github/workflows/cluster_endtoend_vtorc_mysql57.yml index c33b13a1323..dc9f3581a62 100644 --- a/.github/workflows/cluster_endtoend_vtorc_mysql57.yml +++ b/.github/workflows/cluster_endtoend_vtorc_mysql57.yml @@ -43,6 +43,15 @@ jobs: draft=$(echo "$PR_DATA" | jq .draft -r) echo "is_draft=${draft}" >> $GITHUB_OUTPUT + - name: Check Memory + run: | + totalMem=$(free -g | awk 'NR==2 {print $2}') + echo "total memory $totalMem GB" + if [[ "$totalMem" -lt 15 ]]; then + echo "Less memory than required" + exit 1 + fi + - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index c60076e9766..d047d588f2a 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -137,6 +137,9 @@ var ( "vtgate_topo_consul", "tabletmanager_consul", } + clustersRequiringMemoryCheck = []string{ + "vtorc", + } clusterRequiring16CoresMachines = []string{ "onlineddl_vrepl", "onlineddl_vrepl_stress", @@ -154,6 +157,7 @@ type unitTest struct { type clusterTest struct { Name, Shard, Platform string FileName string + MemoryCheck bool MakeTools, InstallXtraBackup bool Docker bool LimitResourceUsage bool @@ -351,6 +355,13 @@ func generateClusterWorkflows(list []string, tpl string) { break } } + memoryCheckClusters := canonnizeList(clustersRequiringMemoryCheck) + for _, memCheckCluster := range memoryCheckClusters { + if memCheckCluster == cluster { + test.MemoryCheck = true + break + } + } xtraBackupClusters := canonnizeList(clustersRequiringXtraBackup) for _, xtraBackupCluster := range xtraBackupClusters { if xtraBackupCluster == cluster { diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index ef1a991c169..d75cdbe817d 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -41,6 +41,19 @@ jobs: draft=$(echo "$PR_DATA" | jq .draft -r) echo "is_draft=${draft}" >> $GITHUB_OUTPUT + {{if .MemoryCheck}} + + - name: Check Memory + run: | + totalMem=$(free -g | awk 'NR==2 {print $2}') + echo "total memory $totalMem GB" + if [[ "$totalMem" -lt 15 ]]; then + echo "Less memory than required" + exit 1 + fi + + {{end}} + - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 diff --git a/test/templates/cluster_endtoend_test_mysql57.tpl b/test/templates/cluster_endtoend_test_mysql57.tpl index 9f9e51fbc69..b4f528eeb2f 100644 --- a/test/templates/cluster_endtoend_test_mysql57.tpl +++ b/test/templates/cluster_endtoend_test_mysql57.tpl @@ -46,6 +46,19 @@ jobs: draft=$(echo "$PR_DATA" | jq .draft -r) echo "is_draft=${draft}" >> $GITHUB_OUTPUT + {{if .MemoryCheck}} + + - name: Check Memory + run: | + totalMem=$(free -g | awk 'NR==2 {print $2}') + echo "total memory $totalMem GB" + if [[ "$totalMem" -lt 15 ]]; then + echo "Less memory than required" + exit 1 + fi + + {{end}} + - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 From a9b2c18a2a5d460ef484e6542bb9b8f3fe548809 Mon Sep 17 00:00:00 2001 From: Max Englander Date: Wed, 21 Feb 2024 11:17:11 -0500 Subject: [PATCH 02/16] planner: support union statements with ctes (#15312) Signed-off-by: Max Englander --- go/vt/sqlparser/ast_format.go | 4 + go/vt/sqlparser/ast_format_fast.go | 4 + go/vt/sqlparser/parse_test.go | 3 + .../planbuilder/testdata/cte_cases.json | 103 ++++++++++++++++++ 4 files changed, 114 insertions(+) diff --git a/go/vt/sqlparser/ast_format.go b/go/vt/sqlparser/ast_format.go index 0e7ca0a8231..e90b8b6344b 100644 --- a/go/vt/sqlparser/ast_format.go +++ b/go/vt/sqlparser/ast_format.go @@ -74,6 +74,10 @@ func (node *CommentOnly) Format(buf *TrackedBuffer) { // Format formats the node. func (node *Union) Format(buf *TrackedBuffer) { + if node.With != nil { + buf.astPrintf(node, "%v", node.With) + } + if requiresParen(node.Left) { buf.astPrintf(node, "(%v)", node.Left) } else { diff --git a/go/vt/sqlparser/ast_format_fast.go b/go/vt/sqlparser/ast_format_fast.go index a7f1a3c1e93..4ebdbcf8475 100644 --- a/go/vt/sqlparser/ast_format_fast.go +++ b/go/vt/sqlparser/ast_format_fast.go @@ -86,6 +86,10 @@ func (node *CommentOnly) FormatFast(buf *TrackedBuffer) { // FormatFast formats the node. func (node *Union) FormatFast(buf *TrackedBuffer) { + if node.With != nil { + node.With.FormatFast(buf) + } + if requiresParen(node.Left) { buf.WriteByte('(') node.Left.FormatFast(buf) diff --git a/go/vt/sqlparser/parse_test.go b/go/vt/sqlparser/parse_test.go index f90ae16606e..0dce5eae5bc 100644 --- a/go/vt/sqlparser/parse_test.go +++ b/go/vt/sqlparser/parse_test.go @@ -505,6 +505,9 @@ var ( }, { input: "WITH topsales2003 AS (SELECT salesRepEmployeeNumber employeeNumber, SUM(quantityOrdered * priceEach) sales FROM orders INNER JOIN orderdetails USING (orderNumber) INNER JOIN customers USING (customerNumber) WHERE YEAR(shippedDate) = 2003 AND status = 'Shipped' GROUP BY salesRepEmployeeNumber ORDER BY sales DESC LIMIT 5)SELECT employeeNumber, firstName, lastName, sales FROM employees JOIN topsales2003 USING (employeeNumber)", output: "with topsales2003 as (select salesRepEmployeeNumber as employeeNumber, sum(quantityOrdered * priceEach) as sales from orders join orderdetails using (orderNumber) join customers using (customerNumber) where YEAR(shippedDate) = 2003 and `status` = 'Shipped' group by salesRepEmployeeNumber order by sales desc limit 5) select employeeNumber, firstName, lastName, sales from employees join topsales2003 using (employeeNumber)", + }, { + input: "WITH count_a AS (SELECT COUNT(`id`) AS `num` FROM `tbl_a`), count_b AS (SELECT COUNT(`id`) AS `num` FROM tbl_b) SELECT 'a', `num` FROM `count_a` UNION SELECT 'b', `num` FROM `count_b`", + output: "with count_a as (select count(id) as num from tbl_a) , count_b as (select count(id) as num from tbl_b) select 'a', num from count_a union select 'b', num from count_b", }, { input: "select 1 from t", }, { diff --git a/go/vt/vtgate/planbuilder/testdata/cte_cases.json b/go/vt/vtgate/planbuilder/testdata/cte_cases.json index e43b6320340..a7027f80348 100644 --- a/go/vt/vtgate/planbuilder/testdata/cte_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/cte_cases.json @@ -1863,5 +1863,108 @@ "main.unsharded" ] } + }, + { + "comment": "WITH two common expressions against an unsharded datbase and a SELECT UNION against those expressions", + "query": "WITH `count_a` AS (SELECT COUNT(`id`) AS `num` FROM `unsharded_a`), `count_b` AS (SELECT COUNT(`id`) AS `num` FROM `unsharded_b`) SELECT 'count_a' AS `tab`, `num` FROM `count_a` UNION SELECT 'count_b' AS `tab`, `num` FROM `count_b`", + "plan": { + "QueryType": "SELECT", + "Original": "WITH `count_a` AS (SELECT COUNT(`id`) AS `num` FROM `unsharded_a`), `count_b` AS (SELECT COUNT(`id`) AS `num` FROM `unsharded_b`) SELECT 'count_a' AS `tab`, `num` FROM `count_a` UNION SELECT 'count_b' AS `tab`, `num` FROM `count_b`", + "Instructions": { + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select 'count_a' as tab, num from count_a where 1 != 1 union select 'count_b' as tab, num from count_b where 1 != 1", + "Query": "with count_a as (select count(id) as num from unsharded_a) , count_b as (select count(id) as num from unsharded_b) select 'count_a' as tab, num from count_a union select 'count_b' as tab, num from count_b", + "Table": "unsharded_a, unsharded_b" + }, + "TablesUsed": [ + "main.unsharded_a", + "main.unsharded_b" + ] + } + }, + { + "comment": "WITH two common expressions against a sharded datbase and a SELECT UNION against those expressions", + "query": "WITH `count_a` AS (SELECT COUNT(`user_id`) AS `num` FROM `user_metadata`), `count_b` AS (SELECT COUNT(`user_id`) AS `num` FROM `user_extra`) SELECT 'count_a' AS `tab`, `num` FROM `count_a` UNION SELECT 'count_b' AS `tab`, `num` FROM `count_b`", + "plan": { + "QueryType": "SELECT", + "Original": "WITH `count_a` AS (SELECT COUNT(`user_id`) AS `num` FROM `user_metadata`), `count_b` AS (SELECT COUNT(`user_id`) AS `num` FROM `user_extra`) SELECT 'count_a' AS `tab`, `num` FROM `count_a` UNION SELECT 'count_b' AS `tab`, `num` FROM `count_b`", + "Instructions": { + "OperatorType": "Distinct", + "Collations": [ + "0: utf8mb4_0900_ai_ci", + "1" + ], + "Inputs": [ + { + "OperatorType": "Concatenate", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + "'count_a' as tab", + ":0 as num" + ], + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Scalar", + "Aggregates": "sum_count(0) AS num", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select count(user_id) as num from user_metadata where 1 != 1", + "Query": "select count(user_id) as num from user_metadata", + "Table": "user_metadata" + } + ] + } + ] + }, + { + "OperatorType": "Projection", + "Expressions": [ + "'count_b' as tab", + ":0 as num" + ], + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Scalar", + "Aggregates": "sum_count(0) AS num", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select count(user_id) as num from user_extra where 1 != 1", + "Query": "select count(user_id) as num from user_extra", + "Table": "user_extra" + } + ] + } + ] + } + ] + } + ] + }, + "TablesUsed": [ + "user.user_extra", + "user.user_metadata" + ] + } } ] From e163c9eee03da72eb38160508b38d87cab572f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicent=20Mart=C3=AD?= <42793+vmg@users.noreply.github.com> Date: Wed, 21 Feb 2024 18:03:39 +0100 Subject: [PATCH 03/16] tablet: remove max-waiters setting (#15323) Signed-off-by: Vicent Marti --- config/tablet/default.yaml | 3 --- doc/design-docs/TabletServerParamsAsYAML.md | 3 --- go/flags/endtoend/vtcombo.txt | 3 --- go/flags/endtoend/vttablet.txt | 3 --- go/vt/vttablet/tabletserver/tabletenv/config.go | 17 +++++++++-------- .../tabletserver/tabletenv/config_test.go | 6 ------ go/vt/vttablet/tabletserver/tx_pool_test.go | 8 -------- 7 files changed, 9 insertions(+), 34 deletions(-) diff --git a/config/tablet/default.yaml b/config/tablet/default.yaml index f996bb04737..ec9d1f94833 100644 --- a/config/tablet/default.yaml +++ b/config/tablet/default.yaml @@ -57,21 +57,18 @@ oltpReadPool: idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout maxLifetimeSeconds: 0 # queryserver-config-pool-conn-max-lifetime prefillParallelism: 0 # queryserver-config-pool-prefill-parallelism - maxWaiters: 50000 # queryserver-config-query-pool-waiter-cap olapReadPool: size: 200 # queryserver-config-stream-pool-size timeoutSeconds: 0 # queryserver-config-query-pool-timeout idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout prefillParallelism: 0 # queryserver-config-stream-pool-prefill-parallelism - maxWaiters: 0 txPool: size: 20 # queryserver-config-transaction-cap timeoutSeconds: 1 # queryserver-config-txpool-timeout idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout prefillParallelism: 0 # queryserver-config-transaction-prefill-parallelism - maxWaiters: 50000 # queryserver-config-txpool-waiter-cap oltp: queryTimeoutSeconds: 30 # queryserver-config-query-timeout diff --git a/doc/design-docs/TabletServerParamsAsYAML.md b/doc/design-docs/TabletServerParamsAsYAML.md index 49d073d1313..52d48a5e6f6 100644 --- a/doc/design-docs/TabletServerParamsAsYAML.md +++ b/doc/design-docs/TabletServerParamsAsYAML.md @@ -95,21 +95,18 @@ oltpReadPool: timeoutSeconds: 0 # queryserver-config-query-pool-timeout idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout prefillParallelism: 0 # queryserver-config-pool-prefill-parallelism - maxWaiters: 50000 # queryserver-config-query-pool-waiter-cap olapReadPool: size: 200 # queryserver-config-stream-pool-size timeoutSeconds: 0 # queryserver-config-query-pool-timeout idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout prefillParallelism: 0 # queryserver-config-stream-pool-prefill-parallelism - maxWaiters: 0 txPool: size: 20 # queryserver-config-transaction-cap timeoutSeconds: 1 # queryserver-config-txpool-timeout idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout prefillParallelism: 0 # queryserver-config-transaction-prefill-parallelism - maxWaiters: 50000 # queryserver-config-txpool-waiter-cap oltp: queryTimeoutSeconds: 30 # queryserver-config-query-timeout diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 59dd8969880..0843ee46563 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -283,21 +283,18 @@ Flags: --queryserver-config-pool-size int query server read pool size, connection pool is used by regular queries (non streaming, not in a transaction) (default 16) --queryserver-config-query-cache-memory int query server query cache size in bytes, maximum amount of memory to be used for caching. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache. (default 33554432) --queryserver-config-query-pool-timeout duration query server query pool timeout, it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead. - --queryserver-config-query-pool-waiter-cap int query server query pool waiter limit, this is the maximum number of queries that can be queued waiting to get a connection (default 5000) --queryserver-config-query-timeout duration query server query timeout, this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed. (default 30s) --queryserver-config-schema-change-signal query server schema signal, will signal connected vtgates that schema has changed whenever this is detected. VTGates will need to have -schema_change_signal enabled for this to work (default true) --queryserver-config-schema-reload-time duration query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time. (default 30m0s) --queryserver-config-stream-buffer-size int query server stream buffer size, the maximum number of bytes sent from vttablet for each stream call. It's recommended to keep this value in sync with vtgate's stream_buffer_size. (default 32768) --queryserver-config-stream-pool-size int query server stream connection pool size, stream pool is used by stream queries: queries that return results to client in a streaming fashion (default 200) --queryserver-config-stream-pool-timeout duration query server stream pool timeout, it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout. - --queryserver-config-stream-pool-waiter-cap int query server stream pool waiter limit, this is the maximum number of streaming queries that can be queued waiting to get a connection --queryserver-config-strict-table-acl only allow queries that pass table acl checks --queryserver-config-terse-errors prevent bind vars from escaping in client error messages --queryserver-config-transaction-cap int query server transaction cap is the maximum number of transactions allowed to happen at any given point of a time for a single vttablet. E.g. by setting transaction cap to 100, there are at most 100 transactions will be processed by a vttablet and the 101th transaction will be blocked (and fail if it cannot get connection within specified timeout) (default 20) --queryserver-config-transaction-timeout duration query server transaction timeout, a transaction will be killed if it takes longer than this value (default 30s) --queryserver-config-truncate-error-len int truncate errors sent to client if they are longer than this value (0 means do not truncate) --queryserver-config-txpool-timeout duration query server transaction pool timeout, it is how long vttablet waits if tx pool is full (default 1s) - --queryserver-config-txpool-waiter-cap int query server transaction pool waiter limit, this is the maximum number of transactions that can be queued waiting to get a connection (default 5000) --queryserver-config-warn-result-size int query server result size warning threshold, warn if number of rows returned from vttablet for non-streaming queries exceeds this --queryserver-enable-settings-pool Enable pooling of connections with modified system settings (default true) --queryserver-enable-views Enable views support in vttablet. diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index ac25179db9b..bb7403ef1cb 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -276,21 +276,18 @@ Flags: --queryserver-config-pool-size int query server read pool size, connection pool is used by regular queries (non streaming, not in a transaction) (default 16) --queryserver-config-query-cache-memory int query server query cache size in bytes, maximum amount of memory to be used for caching. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache. (default 33554432) --queryserver-config-query-pool-timeout duration query server query pool timeout, it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead. - --queryserver-config-query-pool-waiter-cap int query server query pool waiter limit, this is the maximum number of queries that can be queued waiting to get a connection (default 5000) --queryserver-config-query-timeout duration query server query timeout, this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed. (default 30s) --queryserver-config-schema-change-signal query server schema signal, will signal connected vtgates that schema has changed whenever this is detected. VTGates will need to have -schema_change_signal enabled for this to work (default true) --queryserver-config-schema-reload-time duration query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time. (default 30m0s) --queryserver-config-stream-buffer-size int query server stream buffer size, the maximum number of bytes sent from vttablet for each stream call. It's recommended to keep this value in sync with vtgate's stream_buffer_size. (default 32768) --queryserver-config-stream-pool-size int query server stream connection pool size, stream pool is used by stream queries: queries that return results to client in a streaming fashion (default 200) --queryserver-config-stream-pool-timeout duration query server stream pool timeout, it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout. - --queryserver-config-stream-pool-waiter-cap int query server stream pool waiter limit, this is the maximum number of streaming queries that can be queued waiting to get a connection --queryserver-config-strict-table-acl only allow queries that pass table acl checks --queryserver-config-terse-errors prevent bind vars from escaping in client error messages --queryserver-config-transaction-cap int query server transaction cap is the maximum number of transactions allowed to happen at any given point of a time for a single vttablet. E.g. by setting transaction cap to 100, there are at most 100 transactions will be processed by a vttablet and the 101th transaction will be blocked (and fail if it cannot get connection within specified timeout) (default 20) --queryserver-config-transaction-timeout duration query server transaction timeout, a transaction will be killed if it takes longer than this value (default 30s) --queryserver-config-truncate-error-len int truncate errors sent to client if they are longer than this value (0 means do not truncate) --queryserver-config-txpool-timeout duration query server transaction pool timeout, it is how long vttablet waits if tx pool is full (default 1s) - --queryserver-config-txpool-waiter-cap int query server transaction pool waiter limit, this is the maximum number of transactions that can be queued waiting to get a connection (default 5000) --queryserver-config-warn-result-size int query server result size warning threshold, warn if number of rows returned from vttablet for non-streaming queries exceeds this --queryserver-enable-settings-pool Enable pooling of connections with modified system settings (default true) --queryserver-enable-views Enable views support in vttablet. diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 25352aba91b..233f8951227 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -142,9 +142,15 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) { fs.DurationVar(¤tConfig.TxPool.Timeout, "queryserver-config-txpool-timeout", defaultConfig.TxPool.Timeout, "query server transaction pool timeout, it is how long vttablet waits if tx pool is full") fs.DurationVar(¤tConfig.OltpReadPool.IdleTimeout, "queryserver-config-idle-timeout", defaultConfig.OltpReadPool.IdleTimeout, "query server idle timeout, vttablet manages various mysql connection pools. This config means if a connection has not been used in given idle timeout, this connection will be removed from pool. This effectively manages number of connection objects and optimize the pool performance.") fs.DurationVar(¤tConfig.OltpReadPool.MaxLifetime, "queryserver-config-pool-conn-max-lifetime", defaultConfig.OltpReadPool.MaxLifetime, "query server connection max lifetime, vttablet manages various mysql connection pools. This config means if a connection has lived at least this long, it connection will be removed from pool upon the next time it is returned to the pool.") - fs.IntVar(¤tConfig.OltpReadPool.MaxWaiters, "queryserver-config-query-pool-waiter-cap", defaultConfig.OltpReadPool.MaxWaiters, "query server query pool waiter limit, this is the maximum number of queries that can be queued waiting to get a connection") - fs.IntVar(¤tConfig.OlapReadPool.MaxWaiters, "queryserver-config-stream-pool-waiter-cap", defaultConfig.OlapReadPool.MaxWaiters, "query server stream pool waiter limit, this is the maximum number of streaming queries that can be queued waiting to get a connection") - fs.IntVar(¤tConfig.TxPool.MaxWaiters, "queryserver-config-txpool-waiter-cap", defaultConfig.TxPool.MaxWaiters, "query server transaction pool waiter limit, this is the maximum number of transactions that can be queued waiting to get a connection") + + var unused int + fs.IntVar(&unused, "queryserver-config-query-pool-waiter-cap", 0, "query server query pool waiter limit, this is the maximum number of queries that can be queued waiting to get a connection") + fs.IntVar(&unused, "queryserver-config-stream-pool-waiter-cap", 0, "query server stream pool waiter limit, this is the maximum number of streaming queries that can be queued waiting to get a connection") + fs.IntVar(&unused, "queryserver-config-txpool-waiter-cap", 0, "query server transaction pool waiter limit, this is the maximum number of transactions that can be queued waiting to get a connection") + fs.MarkDeprecated("queryserver-config-query-pool-waiter-cap", "The new connection pool in v19 does not limit waiter capacity. This flag will be removed in a future release.") + fs.MarkDeprecated("queryserver-config-stream-pool-waiter-cap", "The new connection pool in v19 does not limit waiter capacity. This flag will be removed in a future release.") + fs.MarkDeprecated("queryserver-config-txpool-waiter-cap", "The new connection pool in v19 does not limit waiter capacity. This flag will be removed in a future release.") + // tableacl related configurations. fs.BoolVar(¤tConfig.StrictTableACL, "queryserver-config-strict-table-acl", defaultConfig.StrictTableACL, "only allow queries that pass table acl checks") fs.BoolVar(¤tConfig.EnableTableACLDryRun, "queryserver-config-enable-table-acl-dry-run", defaultConfig.EnableTableACLDryRun, "If this flag is enabled, tabletserver will emit monitoring metrics and let the request pass regardless of table acl check results") @@ -440,7 +446,6 @@ type ConnPoolConfig struct { IdleTimeout time.Duration `json:"idleTimeoutSeconds,omitempty"` MaxLifetime time.Duration `json:"maxLifetimeSeconds,omitempty"` PrefillParallelism int `json:"prefillParallelism,omitempty"` - MaxWaiters int `json:"maxWaiters,omitempty"` } func (cfg *ConnPoolConfig) MarshalJSON() ([]byte, error) { @@ -477,7 +482,6 @@ func (cfg *ConnPoolConfig) UnmarshalJSON(data []byte) (err error) { IdleTimeout string `json:"idleTimeoutSeconds,omitempty"` MaxLifetime string `json:"maxLifetimeSeconds,omitempty"` PrefillParallelism int `json:"prefillParallelism,omitempty"` - MaxWaiters int `json:"maxWaiters,omitempty"` } if err := json.Unmarshal(data, &tmp); err != nil { @@ -507,7 +511,6 @@ func (cfg *ConnPoolConfig) UnmarshalJSON(data []byte) (err error) { cfg.Size = tmp.Size cfg.PrefillParallelism = tmp.PrefillParallelism - cfg.MaxWaiters = tmp.MaxWaiters return nil } @@ -945,7 +948,6 @@ var defaultConfig = TabletConfig{ OltpReadPool: ConnPoolConfig{ Size: 16, IdleTimeout: 30 * time.Minute, - MaxWaiters: 5000, }, OlapReadPool: ConnPoolConfig{ Size: 200, @@ -955,7 +957,6 @@ var defaultConfig = TabletConfig{ Size: 20, Timeout: time.Second, IdleTimeout: 30 * time.Minute, - MaxWaiters: 5000, }, Olap: OlapConfig{ TxTimeout: 30 * time.Second, diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go index c6f65cb94cb..98d4cfceb21 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go @@ -48,7 +48,6 @@ func TestConfigParse(t *testing.T) { }, OltpReadPool: ConnPoolConfig{ Size: 16, - MaxWaiters: 40, Timeout: 10 * time.Second, IdleTimeout: 20 * time.Second, MaxLifetime: 50 * time.Second, @@ -86,7 +85,6 @@ oltp: {} oltpReadPool: idleTimeoutSeconds: 20s maxLifetimeSeconds: 50s - maxWaiters: 40 size: 16 timeoutSeconds: 10s replicationTracker: {} @@ -109,7 +107,6 @@ txPool: {} oltpReadPool: size: 16 idleTimeoutSeconds: 20s - maxWaiters: 40 maxLifetimeSeconds: 50s `) gotCfg := cfg @@ -148,7 +145,6 @@ oltp: txTimeoutSeconds: 30s oltpReadPool: idleTimeoutSeconds: 30m0s - maxWaiters: 5000 size: 16 queryCacheDoorkeeper: true queryCacheMemory: 33554432 @@ -164,7 +160,6 @@ signalWhenSchemaChange: true streamBufferSize: 32768 txPool: idleTimeoutSeconds: 30m0s - maxWaiters: 5000 size: 20 timeoutSeconds: 1s ` @@ -178,7 +173,6 @@ func TestClone(t *testing.T) { cfg1 := &TabletConfig{ OltpReadPool: ConnPoolConfig{ Size: 16, - MaxWaiters: 40, Timeout: 10 * time.Second, IdleTimeout: 20 * time.Second, MaxLifetime: 50 * time.Second, diff --git a/go/vt/vttablet/tabletserver/tx_pool_test.go b/go/vt/vttablet/tabletserver/tx_pool_test.go index 9edbe5b3592..37500ada79a 100644 --- a/go/vt/vttablet/tabletserver/tx_pool_test.go +++ b/go/vt/vttablet/tabletserver/tx_pool_test.go @@ -305,7 +305,6 @@ func TestTxPoolWaitTimeoutError(t *testing.T) { env := newEnv("TabletServerTest") env.Config().TxPool.Size = 1 - env.Config().TxPool.MaxWaiters = 0 env.Config().TxPool.Timeout = time.Second // given db, txPool, _, closer := setupWithEnv(t, env) @@ -428,7 +427,6 @@ func TestTxTimeoutKillsTransactions(t *testing.T) { env := newEnv("TabletServerTest") env.Config().TxPool.Size = 1 - env.Config().TxPool.MaxWaiters = 0 env.Config().Oltp.TxTimeout = time.Second _, txPool, limiter, closer := setupWithEnv(t, env) defer closer() @@ -477,7 +475,6 @@ func TestTxTimeoutDoesNotKillShortLivedTransactions(t *testing.T) { env := newEnv("TabletServerTest") env.Config().TxPool.Size = 1 - env.Config().TxPool.MaxWaiters = 0 env.Config().Oltp.TxTimeout = time.Second _, txPool, _, closer := setupWithEnv(t, env) defer closer() @@ -510,7 +507,6 @@ func TestTxTimeoutKillsOlapTransactions(t *testing.T) { env := newEnv("TabletServerTest") env.Config().TxPool.Size = 1 - env.Config().TxPool.MaxWaiters = 0 env.Config().Oltp.TxTimeout = time.Second env.Config().Olap.TxTimeout = 2 * time.Second _, txPool, _, closer := setupWithEnv(t, env) @@ -548,7 +544,6 @@ func TestTxTimeoutNotEnforcedForZeroLengthTimeouts(t *testing.T) { env := newEnv("TabletServerTest") env.Config().TxPool.Size = 2 - env.Config().TxPool.MaxWaiters = 0 env.Config().Oltp.TxTimeout = 0 env.Config().Olap.TxTimeout = 0 _, txPool, _, closer := setupWithEnv(t, env) @@ -591,7 +586,6 @@ func TestTxTimeoutReservedConn(t *testing.T) { env := newEnv("TabletServerTest") env.Config().TxPool.Size = 1 - env.Config().TxPool.MaxWaiters = 0 env.Config().Oltp.TxTimeout = time.Second env.Config().Olap.TxTimeout = 2 * time.Second _, txPool, _, closer := setupWithEnv(t, env) @@ -634,7 +628,6 @@ func TestTxTimeoutReusedReservedConn(t *testing.T) { env := newEnv("TabletServerTest") env.Config().TxPool.Size = 1 - env.Config().TxPool.MaxWaiters = 0 env.Config().Oltp.TxTimeout = time.Second env.Config().Olap.TxTimeout = 2 * time.Second _, txPool, _, closer := setupWithEnv(t, env) @@ -820,7 +813,6 @@ func newEnv(exporterName string) tabletenv.Env { cfg.TxPool.Size = 300 cfg.Oltp.TxTimeout = 30 * time.Second cfg.TxPool.Timeout = 40 * time.Second - cfg.TxPool.MaxWaiters = 500000 cfg.OltpReadPool.IdleTimeout = 30 * time.Second cfg.OlapReadPool.IdleTimeout = 30 * time.Second cfg.TxPool.IdleTimeout = 30 * time.Second From 0adb706b5799fb655d013104f9fffcccce5b4831 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 21 Feb 2024 15:32:35 -0500 Subject: [PATCH 04/16] [e2e] More vtctldclient updates in tests (#15276) Signed-off-by: Andrew Mason --- .../endtoend/cluster/vtctldclient_process.go | 51 ++++++++++++++++++ go/test/endtoend/keyspace/keyspace_test.go | 52 +++++++++--------- .../reparent/newfeaturetest/reparent_test.go | 6 +-- .../reparent/plannedreparent/reparent_test.go | 26 ++++----- go/test/endtoend/reparent/utils/utils.go | 53 ++++++++----------- .../endtoend/sharded/sharded_keyspace_test.go | 18 +++---- .../buffer/reparent/failover_buffer_test.go | 6 +-- .../buffer/reshard/sharded_buffer_test.go | 9 ++-- .../endtoend/tabletmanager/commands_test.go | 37 +++++++------ .../tabletmanager/custom_rule_topo_test.go | 2 +- .../tabletmanager/primary/tablet_test.go | 23 ++++---- .../tabletmanager/tablet_health_test.go | 43 +++++++-------- go/test/endtoend/tabletmanager/tablet_test.go | 6 +-- .../throttler_topo/throttler_test.go | 8 +-- go/test/endtoend/topoconncache/main_test.go | 12 ++--- .../topoconncache/topo_conn_cache_test.go | 4 +- go/test/endtoend/vault/vault_test.go | 2 +- .../endtoend/versionupgrade/upgrade_test.go | 2 +- .../vtgate/createdb_plugin/main_test.go | 4 +- .../endtoend/vtgate/foreignkey/main_test.go | 2 +- .../foreignkey/stress/fk_stress_test.go | 8 +-- go/test/endtoend/vtgate/gen4/main_test.go | 4 +- go/test/endtoend/vtgate/main_test.go | 4 +- .../queries/informationschema/main_test.go | 4 +- .../queries/lookup_queries/main_test.go | 2 +- .../reservedconn/reconnect1/main_test.go | 12 ++--- .../reservedconn/reconnect2/main_test.go | 4 +- .../reservedconn/reconnect3/main_test.go | 2 +- .../reservedconn/reconnect4/main_test.go | 2 +- go/test/endtoend/vtgate/schema/schema_test.go | 30 +++++------ .../sharded_prs/st_sharded_test.go | 2 +- .../tablet_healthcheck/reparent_test.go | 6 +-- .../correctness_test.go | 2 +- go/test/endtoend/vtorc/api/api_test.go | 2 +- go/test/endtoend/vtorc/general/vtorc_test.go | 14 ++--- .../primaryfailure/primary_failure_test.go | 14 ++--- go/test/endtoend/vtorc/utils/utils.go | 24 +++------ 37 files changed, 262 insertions(+), 240 deletions(-) diff --git a/go/test/endtoend/cluster/vtctldclient_process.go b/go/test/endtoend/cluster/vtctldclient_process.go index 959ab5a93b9..c5afd8f1220 100644 --- a/go/test/endtoend/cluster/vtctldclient_process.go +++ b/go/test/endtoend/cluster/vtctldclient_process.go @@ -27,6 +27,7 @@ import ( "vitess.io/vitess/go/vt/vterrors" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) // VtctldClientProcess is a generic handle for a running vtctldclient command . @@ -97,6 +98,11 @@ func VtctldClientProcessInstance(hostname string, grpcPort int, tmpDirectory str return vtctldclient } +// ApplyRoutingRules applies the given routing rules. +func (vtctldclient *VtctldClientProcess) ApplyRoutingRules(json string) error { + return vtctldclient.ExecuteCommand("ApplyRoutingRules", "--rules", json) +} + type ApplySchemaParams struct { DDLStrategy string MigrationContext string @@ -213,6 +219,51 @@ func (vtctldclient *VtctldClientProcess) CreateKeyspace(keyspaceName string, sid return err } +// GetKeyspace executes the vtctldclient command to get a shard, and parses the response. +func (vtctldclient *VtctldClientProcess) GetKeyspace(keyspace string) (*vtctldatapb.Keyspace, error) { + data, err := vtctldclient.ExecuteCommandWithOutput("GetKeyspace", keyspace) + if err != nil { + return nil, err + } + + var ks vtctldatapb.Keyspace + err = json2.Unmarshal([]byte(data), &ks) + if err != nil { + return nil, vterrors.Wrapf(err, "failed to parse keyspace output: %s", data) + } + return &ks, nil +} + +// GetShard executes the vtctldclient command to get a shard, and parses the response. +func (vtctldclient *VtctldClientProcess) GetShard(keyspace string, shard string) (*vtctldatapb.Shard, error) { + data, err := vtctldclient.ExecuteCommandWithOutput("GetShard", fmt.Sprintf("%s/%s", keyspace, shard)) + if err != nil { + return nil, err + } + + var si vtctldatapb.Shard + err = json2.Unmarshal([]byte(data), &si) + if err != nil { + return nil, vterrors.Wrapf(err, "failed to parse shard output: %s", data) + } + return &si, nil +} + +// GetTablet executes vtctldclient command to get a tablet, and parses the response. +func (vtctldclient *VtctldClientProcess) GetTablet(alias string) (*topodatapb.Tablet, error) { + data, err := vtctldclient.ExecuteCommandWithOutput("GetTablet", alias) + if err != nil { + return nil, err + } + + var tablet topodatapb.Tablet + err = json2.Unmarshal([]byte(data), &tablet) + if err != nil { + return nil, vterrors.Wrapf(err, "failed to parse tablet output: %s", data) + } + return &tablet, nil +} + // OnlineDDLShowRecent responds with recent schema migration list func (vtctldclient *VtctldClientProcess) OnlineDDLShowRecent(Keyspace string) (result string, err error) { return vtctldclient.ExecuteCommandWithOutput( diff --git a/go/test/endtoend/keyspace/keyspace_test.go b/go/test/endtoend/keyspace/keyspace_test.go index 142e4b4b442..7f7d4198135 100644 --- a/go/test/endtoend/keyspace/keyspace_test.go +++ b/go/test/endtoend/keyspace/keyspace_test.go @@ -21,17 +21,18 @@ import ( "encoding/json" "flag" "os" - "strings" "testing" - "vitess.io/vitess/go/constants/sidecar" - "vitess.io/vitess/go/vt/key" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/constants/sidecar" + "vitess.io/vitess/go/json2" "vitess.io/vitess/go/test/endtoend/cluster" - "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/key" + + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) var ( @@ -41,7 +42,7 @@ var ( cell = "zone1" cell2 = "zone2" hostname = "localhost" - servedTypes = map[topodata.TabletType]bool{topodata.TabletType_PRIMARY: true, topodata.TabletType_REPLICA: true, topodata.TabletType_RDONLY: true} + servedTypes = map[topodatapb.TabletType]bool{topodatapb.TabletType_PRIMARY: true, topodatapb.TabletType_REPLICA: true, topodatapb.TabletType_RDONLY: true} sqlSchema = `create table vt_insert_test ( id bigint auto_increment, msg varchar(64), @@ -152,29 +153,31 @@ func TestDurabilityPolicyField(t *testing.T) { out, err = vtctldClientProcess.ExecuteCommandWithOutput("DeleteKeyspace", "ks_durability") require.NoError(t, err, out) - out, err = clusterForKSTest.VtctlProcess.ExecuteCommandWithOutput("CreateKeyspace", "--", "--durability-policy=semi_sync", "ks_durability") + out, err = clusterForKSTest.VtctldClientProcess.ExecuteCommandWithOutput("CreateKeyspace", "--durability-policy=semi_sync", "ks_durability") require.NoError(t, err, out) checkDurabilityPolicy(t, "semi_sync") - out, err = clusterForKSTest.VtctlProcess.ExecuteCommandWithOutput("DeleteKeyspace", "ks_durability") + out, err = clusterForKSTest.VtctldClientProcess.ExecuteCommandWithOutput("DeleteKeyspace", "ks_durability") require.NoError(t, err, out) } func checkDurabilityPolicy(t *testing.T, durabilityPolicy string) { - var keyspace topodata.Keyspace - out, err := clusterForKSTest.VtctlclientProcess.ExecuteCommandWithOutput("GetKeyspace", "ks_durability") - require.NoError(t, err, out) - err = json.Unmarshal([]byte(out), &keyspace) + ks, err := clusterForKSTest.VtctldClientProcess.GetKeyspace("ks_durability") require.NoError(t, err) - require.Equal(t, keyspace.DurabilityPolicy, durabilityPolicy) + require.Equal(t, ks.Keyspace.DurabilityPolicy, durabilityPolicy) } func TestGetSrvKeyspaceNames(t *testing.T) { defer cluster.PanicHandler(t) - output, err := clusterForKSTest.VtctlclientProcess.ExecuteCommandWithOutput("GetSrvKeyspaceNames", cell) + data, err := clusterForKSTest.VtctldClientProcess.ExecuteCommandWithOutput("GetSrvKeyspaceNames", cell) require.Nil(t, err) - assert.Contains(t, strings.Split(output, "\n"), keyspaceUnshardedName) - assert.Contains(t, strings.Split(output, "\n"), keyspaceShardedName) + + var namesByCell = map[string]*vtctldatapb.GetSrvKeyspaceNamesResponse_NameList{} + err = json2.Unmarshal([]byte(data), &namesByCell) + require.NoError(t, err) + + assert.Contains(t, namesByCell[cell].Names, keyspaceUnshardedName) + assert.Contains(t, namesByCell[cell].Names, keyspaceShardedName) } func TestGetSrvKeyspacePartitions(t *testing.T) { @@ -210,7 +213,7 @@ func TestShardNames(t *testing.T) { defer cluster.PanicHandler(t) output, err := clusterForKSTest.VtctlclientProcess.ExecuteCommandWithOutput("GetSrvKeyspace", cell, keyspaceShardedName) require.Nil(t, err) - var srvKeyspace topodata.SrvKeyspace + var srvKeyspace topodatapb.SrvKeyspace err = json.Unmarshal([]byte(output), &srvKeyspace) require.Nil(t, err) @@ -218,12 +221,7 @@ func TestShardNames(t *testing.T) { func TestGetKeyspace(t *testing.T) { defer cluster.PanicHandler(t) - output, err := clusterForKSTest.VtctlclientProcess.ExecuteCommandWithOutput("GetKeyspace", keyspaceUnshardedName) - require.Nil(t, err) - - var keyspace topodata.Keyspace - - err = json.Unmarshal([]byte(output), &keyspace) + _, err := clusterForKSTest.VtctldClientProcess.GetKeyspace(keyspaceUnshardedName) require.Nil(t, err) } @@ -390,7 +388,7 @@ func TestKeyspaceToShardName(t *testing.T) { // for each served type PRIMARY REPLICA RDONLY, the shard ref count should match for _, partition := range srvKeyspace.Partitions { - if partition.ServedType == topodata.TabletType_PRIMARY { + if partition.ServedType == topodatapb.TabletType_PRIMARY { for _, shardRef := range partition.ShardReferences { shardKIDs := shardKIdMap[shardRef.Name] for _, kid := range shardKIDs { @@ -405,7 +403,7 @@ func TestKeyspaceToShardName(t *testing.T) { srvKeyspace = getSrvKeyspace(t, cell, keyspaceUnshardedName) for _, partition := range srvKeyspace.Partitions { - if partition.ServedType == topodata.TabletType_PRIMARY { + if partition.ServedType == topodatapb.TabletType_PRIMARY { for _, shardRef := range partition.ShardReferences { assert.Equal(t, shardRef.Name, keyspaceUnshardedName) } @@ -420,10 +418,10 @@ func packKeyspaceID(keyspaceID uint64) []byte { return (keybytes[:]) } -func getSrvKeyspace(t *testing.T, cell string, ksname string) *topodata.SrvKeyspace { +func getSrvKeyspace(t *testing.T, cell string, ksname string) *topodatapb.SrvKeyspace { output, err := clusterForKSTest.VtctlclientProcess.ExecuteCommandWithOutput("GetSrvKeyspace", cell, ksname) require.Nil(t, err) - var srvKeyspace topodata.SrvKeyspace + var srvKeyspace topodatapb.SrvKeyspace err = json.Unmarshal([]byte(output), &srvKeyspace) require.Nil(t, err) diff --git a/go/test/endtoend/reparent/newfeaturetest/reparent_test.go b/go/test/endtoend/reparent/newfeaturetest/reparent_test.go index b570509f1a7..44cd89a306a 100644 --- a/go/test/endtoend/reparent/newfeaturetest/reparent_test.go +++ b/go/test/endtoend/reparent/newfeaturetest/reparent_test.go @@ -40,7 +40,7 @@ func TestRecoverWithMultipleVttabletFailures(t *testing.T) { utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) // make tablets[1] a rdonly tablet. - err := clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", tablets[1].Alias, "rdonly") + err := clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", tablets[1].Alias, "rdonly") require.NoError(t, err) // Confirm that replication is still working as intended @@ -139,10 +139,10 @@ func TestChangeTypeWithoutSemiSync(t *testing.T) { utils.CheckPrimaryTablet(t, clusterInstance, primary) // Change replica's type to rdonly - err := clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", replica.Alias, "rdonly") + err := clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", replica.Alias, "rdonly") require.NoError(t, err) // Change tablets type from rdonly back to replica - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", replica.Alias, "replica") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", replica.Alias, "replica") require.NoError(t, err) } diff --git a/go/test/endtoend/reparent/plannedreparent/reparent_test.go b/go/test/endtoend/reparent/plannedreparent/reparent_test.go index 6aa5972b928..07e488b52d0 100644 --- a/go/test/endtoend/reparent/plannedreparent/reparent_test.go +++ b/go/test/endtoend/reparent/plannedreparent/reparent_test.go @@ -44,7 +44,7 @@ func TestPrimaryToSpareStateChangeImpossible(t *testing.T) { tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets // We cannot change a primary to spare - out, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ChangeTabletType", tablets[0].Alias, "spare") + out, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("ChangeTabletType", tablets[0].Alias, "spare") require.Error(t, err, out) require.Contains(t, out, "type change PRIMARY -> SPARE is not an allowed transition for ChangeTabletType") } @@ -92,7 +92,7 @@ func TestPRSWithDrainedLaggingTablet(t *testing.T) { defer utils.TeardownCluster(clusterInstance) tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets - err := clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", tablets[1].Alias, "drained") + err := clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", tablets[1].Alias, "drained") require.NoError(t, err) utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) @@ -258,13 +258,13 @@ func reparentFromOutside(t *testing.T, clusterInstance *cluster.LocalProcessClus if downPrimary { err := tablets[0].VttabletProcess.TearDownWithTimeout(30 * time.Second) require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", "--", - "--allow_primary", tablets[0].Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("DeleteTablets", + "--allow-primary", tablets[0].Alias) require.NoError(t, err) } // update topology with the new server - err := clusterInstance.VtctlclientProcess.ExecuteCommand("TabletExternallyReparented", + err := clusterInstance.VtctldClientProcess.ExecuteCommand("TabletExternallyReparented", tablets[1].Alias) require.NoError(t, err) @@ -318,7 +318,7 @@ func TestReparentWithDownReplica(t *testing.T) { // We have to StartReplication on tablets[2] since the MySQL instance is restarted and does not have replication running // We earlier used to rely on replicationManager to fix this but we have disabled it in our testing environment for latest versions of vttablet and vtctl. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", tablets[2].Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("StartReplication", tablets[2].Alias) require.NoError(t, err) // wait until it gets the data @@ -338,9 +338,9 @@ func TestChangeTypeSemiSync(t *testing.T) { primary, replica, rdonly1, rdonly2 := tablets[0], tablets[1], tablets[2], tablets[3] // Updated rdonly tablet and set tablet type to rdonly - err := clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonly1.Alias, "rdonly") + err := clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonly1.Alias, "rdonly") require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonly2.Alias, "rdonly") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonly2.Alias, "rdonly") require.NoError(t, err) utils.ValidateTopology(t, clusterInstance, true) @@ -349,7 +349,7 @@ func TestChangeTypeSemiSync(t *testing.T) { // Stop replication on rdonly1, to make sure when we make it replica it doesn't start again. // Note we do a similar test for replica -> rdonly below. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", rdonly1.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("StopReplication", rdonly1.Alias) require.NoError(t, err) // Check semi-sync on replicas. @@ -364,27 +364,27 @@ func TestChangeTypeSemiSync(t *testing.T) { utils.CheckDBstatus(ctx, t, rdonly2, "Rpl_semi_sync_slave_status", "OFF") // Change replica to rdonly while replicating, should turn off semi-sync, and restart replication. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", replica.Alias, "rdonly") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", replica.Alias, "rdonly") require.NoError(t, err) utils.CheckDBvar(ctx, t, replica, "rpl_semi_sync_slave_enabled", "OFF") utils.CheckDBstatus(ctx, t, replica, "Rpl_semi_sync_slave_status", "OFF") // Change rdonly1 to replica, should turn on semi-sync, and not start replication. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonly1.Alias, "replica") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonly1.Alias, "replica") require.NoError(t, err) utils.CheckDBvar(ctx, t, rdonly1, "rpl_semi_sync_slave_enabled", "ON") utils.CheckDBstatus(ctx, t, rdonly1, "Rpl_semi_sync_slave_status", "OFF") utils.CheckReplicaStatus(ctx, t, rdonly1) // Now change from replica back to rdonly, make sure replication is still not enabled. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonly1.Alias, "rdonly") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonly1.Alias, "rdonly") require.NoError(t, err) utils.CheckDBvar(ctx, t, rdonly1, "rpl_semi_sync_slave_enabled", "OFF") utils.CheckDBstatus(ctx, t, rdonly1, "Rpl_semi_sync_slave_status", "OFF") utils.CheckReplicaStatus(ctx, t, rdonly1) // Change rdonly2 to replica, should turn on semi-sync, and restart replication. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonly2.Alias, "replica") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonly2.Alias, "replica") require.NoError(t, err) utils.CheckDBvar(ctx, t, rdonly2, "rpl_semi_sync_slave_enabled", "ON") utils.CheckDBstatus(ctx, t, rdonly2, "Rpl_semi_sync_slave_status", "ON") diff --git a/go/test/endtoend/reparent/utils/utils.go b/go/test/endtoend/reparent/utils/utils.go index 675648dcf37..790fd0028e2 100644 --- a/go/test/endtoend/reparent/utils/utils.go +++ b/go/test/endtoend/reparent/utils/utils.go @@ -34,7 +34,6 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/vttablet/tabletconn" - "vitess.io/vitess/go/json2" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" @@ -185,7 +184,7 @@ func setupShard(ctx context.Context, t *testing.T, clusterInstance *cluster.Loca } // Initialize shard - err := clusterInstance.VtctlclientProcess.InitializeShard(KeyspaceName, shardName, tablets[0].Cell, tablets[0].TabletUID) + err := clusterInstance.VtctldClientProcess.InitializeShard(KeyspaceName, shardName, tablets[0].Cell, tablets[0].TabletUID) require.NoError(t, err) ValidateTopology(t, clusterInstance, true) @@ -306,21 +305,21 @@ func PrsAvoid(t *testing.T, clusterInstance *cluster.LocalProcessCluster, tab *c // PrsWithTimeout runs PRS func PrsWithTimeout(t *testing.T, clusterInstance *cluster.LocalProcessCluster, tab *cluster.Vttablet, avoid bool, actionTimeout, waitTimeout string) (string, error) { args := []string{ - "PlannedReparentShard", "--", - "--keyspace_shard", fmt.Sprintf("%s/%s", KeyspaceName, ShardName)} + "PlannedReparentShard", + fmt.Sprintf("%s/%s", KeyspaceName, ShardName)} if actionTimeout != "" { args = append(args, "--action_timeout", actionTimeout) } if waitTimeout != "" { - args = append(args, "--wait_replicas_timeout", waitTimeout) + args = append(args, "--wait-replicas-timeout", waitTimeout) } if avoid { - args = append(args, "--avoid_tablet") + args = append(args, "--avoid-primary") } else { - args = append(args, "--new_primary") + args = append(args, "--new-primary") } args = append(args, tab.Alias) - out, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput(args...) + out, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput(args...) return out, err } @@ -335,15 +334,15 @@ func ErsIgnoreTablet(clusterInstance *cluster.LocalProcessCluster, tab *cluster. if timeout != "" { args = append(args, "--action_timeout", timeout) } - args = append(args, "EmergencyReparentShard", "--", "--keyspace_shard", fmt.Sprintf("%s/%s", KeyspaceName, ShardName)) + args = append(args, "EmergencyReparentShard", fmt.Sprintf("%s/%s", KeyspaceName, ShardName)) if tab != nil { - args = append(args, "--new_primary", tab.Alias) + args = append(args, "--new-primary", tab.Alias) } if waitReplicasTimeout != "" { - args = append(args, "--wait_replicas_timeout", waitReplicasTimeout) + args = append(args, "--wait-replicas-timeout", waitReplicasTimeout) } if preventCrossCellPromotion { - args = append(args, "--prevent_cross_cell_promotion=true") + args = append(args, "--prevent-cross-cell-promotion") } if len(tabletsToIgnore) != 0 { tabsString := "" @@ -354,9 +353,9 @@ func ErsIgnoreTablet(clusterInstance *cluster.LocalProcessCluster, tab *cluster. tabsString = tabsString + "," + vttablet.Alias } } - args = append(args, "--ignore_replicas", tabsString) + args = append(args, "--ignore-replicas", tabsString) } - return clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput(args...) + return clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput(args...) } // ErsWithVtctl runs ERS via vtctl binary @@ -374,10 +373,10 @@ func ValidateTopology(t *testing.T, clusterInstance *cluster.LocalProcessCluster args := []string{"Validate"} if pingTablets { - args = append(args, "--", "--ping-tablets=true") + args = append(args, "--ping-tablets") } - out, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput(args...) - require.Empty(t, out) + out, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput(args...) + require.Contains(t, out, "no issues found") require.NoError(t, err) } @@ -398,17 +397,14 @@ func ConfirmReplication(t *testing.T, primary *cluster.Vttablet, replicas []*clu // ConfirmOldPrimaryIsHangingAround confirms that the old primary is hanging around func ConfirmOldPrimaryIsHangingAround(t *testing.T, clusterInstance *cluster.LocalProcessCluster) { - out, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("Validate") + out, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("Validate") require.Error(t, err) require.Contains(t, out, "already has primary") } // CheckPrimaryTablet makes sure the tablet type is primary, and its health check agrees. func CheckPrimaryTablet(t *testing.T, clusterInstance *cluster.LocalProcessCluster, tablet *cluster.Vttablet) { - result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", tablet.Alias) - require.NoError(t, err) - var tabletInfo topodatapb.Tablet - err = json2.Unmarshal([]byte(result), &tabletInfo) + tabletInfo, err := clusterInstance.VtctldClientProcess.GetTablet(tablet.Alias) require.NoError(t, err) assert.Equal(t, topodatapb.TabletType_PRIMARY, tabletInfo.GetType()) @@ -424,10 +420,7 @@ func CheckPrimaryTablet(t *testing.T, clusterInstance *cluster.LocalProcessClust // isHealthyPrimaryTablet will return if tablet is primary AND healthy. func isHealthyPrimaryTablet(t *testing.T, clusterInstance *cluster.LocalProcessCluster, tablet *cluster.Vttablet) bool { - result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", tablet.Alias) - require.Nil(t, err) - var tabletInfo topodatapb.Tablet - err = json2.Unmarshal([]byte(result), &tabletInfo) + tabletInfo, err := clusterInstance.VtctldClientProcess.GetTablet(tablet.Alias) require.Nil(t, err) if tabletInfo.GetType() != topodatapb.TabletType_PRIMARY { return false @@ -541,9 +534,9 @@ func ResurrectTablet(ctx context.Context, t *testing.T, clusterInstance *cluster // DeleteTablet is used to delete the given tablet func DeleteTablet(t *testing.T, clusterInstance *cluster.LocalProcessCluster, tab *cluster.Vttablet) { - err := clusterInstance.VtctlclientProcess.ExecuteCommand( - "DeleteTablet", "--", - "--allow_primary", + err := clusterInstance.VtctldClientProcess.ExecuteCommand( + "DeleteTablets", + "--allow-primary", tab.Alias) require.NoError(t, err) } @@ -629,7 +622,7 @@ func CheckReparentFromOutside(t *testing.T, clusterInstance *cluster.LocalProces // make sure the primary health stream says it's the primary too // (health check is disabled on these servers, force it first) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", tablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", tablet.Alias) require.NoError(t, err) shrs, err := clusterInstance.StreamTabletHealth(context.Background(), tablet, 1) diff --git a/go/test/endtoend/sharded/sharded_keyspace_test.go b/go/test/endtoend/sharded/sharded_keyspace_test.go index 857dc455206..f311404ad7e 100644 --- a/go/test/endtoend/sharded/sharded_keyspace_test.go +++ b/go/test/endtoend/sharded/sharded_keyspace_test.go @@ -108,9 +108,9 @@ func TestShardedKeyspace(t *testing.T) { shard1Primary := shard1.Vttablets[0] shard2Primary := shard2.Vttablets[0] - err := clusterInstance.VtctlclientProcess.InitializeShard(keyspaceName, shard1.Name, cell, shard1Primary.TabletUID) + err := clusterInstance.VtctldClientProcess.InitializeShard(keyspaceName, shard1.Name, cell, shard1Primary.TabletUID) require.Nil(t, err) - err = clusterInstance.VtctlclientProcess.InitializeShard(keyspaceName, shard2.Name, cell, shard2Primary.TabletUID) + err = clusterInstance.VtctldClientProcess.InitializeShard(keyspaceName, shard2.Name, cell, shard2Primary.TabletUID) require.Nil(t, err) err = clusterInstance.StartVTOrc(keyspaceName) @@ -125,7 +125,7 @@ func TestShardedKeyspace(t *testing.T) { _, err = shard2Primary.VttabletProcess.QueryTablet(sqlSchemaReverse, keyspaceName, true) require.Nil(t, err) - if err = clusterInstance.VtctlclientProcess.ApplyVSchema(keyspaceName, vSchema); err != nil { + if err = clusterInstance.VtctldClientProcess.ApplyVSchema(keyspaceName, vSchema); err != nil { log.Error(err.Error()) return } @@ -136,13 +136,13 @@ func TestShardedKeyspace(t *testing.T) { shard2Primary.Alias, shard2.Vttablets[1].Alias) - _ = clusterInstance.VtctlclientProcess.ExecuteCommand("SetReadWrite", shard1Primary.Alias) - _ = clusterInstance.VtctlclientProcess.ExecuteCommand("SetReadWrite", shard2Primary.Alias) + _ = clusterInstance.VtctldClientProcess.ExecuteCommand("SetWritable", shard1Primary.Alias, "true") + _ = clusterInstance.VtctldClientProcess.ExecuteCommand("SetWritable", shard2Primary.Alias, "true") _, _ = shard1Primary.VttabletProcess.QueryTablet("insert into vt_select_test (id, msg) values (1, 'test 1')", keyspaceName, true) _, _ = shard2Primary.VttabletProcess.QueryTablet("insert into vt_select_test (id, msg) values (10, 'test 10')", keyspaceName, true) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Validate", "--", "--ping-tablets") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("Validate", "--ping-tablets") require.Nil(t, err) rows, err := shard1Primary.VttabletProcess.QueryTablet("select id, msg from vt_select_test order by id", keyspaceName, true) @@ -164,9 +164,9 @@ func TestShardedKeyspace(t *testing.T) { assert.Contains(t, output, shard1Primary.Alias+": CREATE TABLE") assert.Contains(t, output, shard2Primary.Alias+": CREATE TABLE") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidateVersionShard", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name)) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ValidateVersionShard", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name)) require.Nil(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("GetPermissions", shard1.Vttablets[1].Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("GetPermissions", shard1.Vttablets[1].Alias) require.Nil(t, err) err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidatePermissionsShard", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name)) require.Nil(t, err) @@ -184,7 +184,7 @@ func TestShardedKeyspace(t *testing.T) { func reloadSchemas(t *testing.T, aliases ...string) { for _, alias := range aliases { - if err := clusterInstance.VtctlclientProcess.ExecuteCommand("ReloadSchema", alias); err != nil { + if err := clusterInstance.VtctldClientProcess.ExecuteCommand("ReloadSchema", alias); err != nil { assert.Fail(t, "Unable to reload schema") } diff --git a/go/test/endtoend/tabletgateway/buffer/reparent/failover_buffer_test.go b/go/test/endtoend/tabletgateway/buffer/reparent/failover_buffer_test.go index 2be57120050..486dc3ef9e5 100644 --- a/go/test/endtoend/tabletgateway/buffer/reparent/failover_buffer_test.go +++ b/go/test/endtoend/tabletgateway/buffer/reparent/failover_buffer_test.go @@ -83,7 +83,7 @@ func failoverExternalReparenting(t *testing.T, clusterInstance *cluster.LocalPro require.NoError(t, err) // Notify the new vttablet primary about the reparent. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("TabletExternallyReparented", newPrimary.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("TabletExternallyReparented", newPrimary.Alias) require.NoError(t, err) } @@ -92,9 +92,9 @@ func failoverPlannedReparenting(t *testing.T, clusterInstance *cluster.LocalProc reads.ExpectQueries(10) writes.ExpectQueries(10) - err := clusterInstance.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", "--", "--keyspace_shard", + err := clusterInstance.VtctldClientProcess.ExecuteCommand("PlannedReparentShard", fmt.Sprintf("%s/%s", keyspaceUnshardedName, "0"), - "--new_primary", clusterInstance.Keyspaces[0].Shards[0].Vttablets[1].Alias) + "--new-primary", clusterInstance.Keyspaces[0].Shards[0].Vttablets[1].Alias) require.NoError(t, err) } diff --git a/go/test/endtoend/tabletgateway/buffer/reshard/sharded_buffer_test.go b/go/test/endtoend/tabletgateway/buffer/reshard/sharded_buffer_test.go index ae922108012..d58d8901165 100644 --- a/go/test/endtoend/tabletgateway/buffer/reshard/sharded_buffer_test.go +++ b/go/test/endtoend/tabletgateway/buffer/reshard/sharded_buffer_test.go @@ -68,9 +68,8 @@ func reshard02(t *testing.T, clusterInstance *cluster.LocalProcessCluster, keysp err := clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, false) require.NoError(t, err) workflowName := "buf2buf" - workflow := fmt.Sprintf("%s.%s", keyspaceName, "buf2buf") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Reshard", "--", "--source_shards", "0", "--target_shards", "-80,80-", "Create", workflow) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("Reshard", "Create", "--target-keyspace", keyspaceName, "--workflow", workflowName, "--source-shards", "0", "--target-shards", "-80,80-") require.NoError(t, err) // Execute the resharding operation @@ -78,13 +77,13 @@ func reshard02(t *testing.T, clusterInstance *cluster.LocalProcessCluster, keysp writes.ExpectQueries(25) waitForLowLag(t, clusterInstance, keyspaceName, workflowName) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Reshard", "--", "--tablet_types=rdonly,replica", "SwitchTraffic", workflow) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("Reshard", "SwitchTraffic", "--target-keyspace", keyspaceName, "--workflow", workflowName, "--tablet-types=rdonly,replica") require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Reshard", "--", "--tablet_types=primary", "SwitchTraffic", workflow) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("Reshard", "SwitchTraffic", "--target-keyspace", keyspaceName, "--workflow", workflowName, "--tablet-types=primary") require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Reshard", "Complete", workflow) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("Reshard", "--target-keyspace", keyspaceName, "--workflow", workflowName, "Complete") require.NoError(t, err) } diff --git a/go/test/endtoend/tabletmanager/commands_test.go b/go/test/endtoend/tabletmanager/commands_test.go index 1a2d2424cb4..537a3b9d0fc 100644 --- a/go/test/endtoend/tabletmanager/commands_test.go +++ b/go/test/endtoend/tabletmanager/commands_test.go @@ -62,52 +62,52 @@ func TestTabletCommands(t *testing.T) { // make sure direct dba queries work sql := "select * from t1" - result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ExecuteFetchAsDba", "--", "--json", primaryTablet.Alias, sql) + result, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("ExecuteFetchAsDBA", "--json", primaryTablet.Alias, sql) require.Nil(t, err) assertExecuteFetch(t, result) // check Ping / RefreshState / RefreshStateByShard - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Ping", primaryTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("PingTablet", primaryTablet.Alias) require.Nil(t, err, "error should be Nil") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RefreshState", primaryTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RefreshState", primaryTablet.Alias) require.Nil(t, err, "error should be Nil") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RefreshStateByShard", keyspaceShard) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RefreshStateByShard", keyspaceShard) require.Nil(t, err, "error should be Nil") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RefreshStateByShard", "--", "--cells="+cell, keyspaceShard) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RefreshStateByShard", "--cells", cell, keyspaceShard) require.Nil(t, err, "error should be Nil") // Check basic actions. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("SetReadOnly", primaryTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("SetWritable", primaryTablet.Alias, "false") require.Nil(t, err, "error should be Nil") qr := utils.Exec(t, conn, "show variables like 'read_only'") got := fmt.Sprintf("%v", qr.Rows) want := "[[VARCHAR(\"read_only\") VARCHAR(\"ON\")]]" assert.Equal(t, want, got) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("SetReadWrite", primaryTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("SetWritable", primaryTablet.Alias, "true") require.Nil(t, err, "error should be Nil") qr = utils.Exec(t, conn, "show variables like 'read_only'") got = fmt.Sprintf("%v", qr.Rows) want = "[[VARCHAR(\"read_only\") VARCHAR(\"OFF\")]]" assert.Equal(t, want, got) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Validate") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("Validate") require.Nil(t, err, "error should be Nil") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Validate", "--", "--ping-tablets=true") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("Validate", "--ping-tablets") require.Nil(t, err, "error should be Nil") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidateKeyspace", keyspaceName) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ValidateKeyspace", keyspaceName) require.Nil(t, err, "error should be Nil") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidateKeyspace", "--", "--ping-tablets=true", keyspaceName) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ValidateKeyspace", "--ping-tablets", keyspaceName) require.Nil(t, err, "error should be Nil") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidateShard", "--", "--ping-tablets=false", keyspaceShard) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ValidateShard", "--ping-tablets", keyspaceShard) require.Nil(t, err, "error should be Nil") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidateShard", "--", "--ping-tablets=true", keyspaceShard) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ValidateShard", "--ping-tablets", keyspaceShard) require.Nil(t, err, "error should be Nil") } @@ -142,14 +142,13 @@ func assertExecuteFetch(t *testing.T, qr string) { // ActionAndTimeout test func TestActionAndTimeout(t *testing.T) { - defer cluster.PanicHandler(t) - err := clusterInstance.VtctlclientProcess.ExecuteCommand("Sleep", primaryTablet.Alias, "5s") + err := clusterInstance.VtctldClientProcess.ExecuteCommand("SleepTablet", primaryTablet.Alias, "5s") require.Nil(t, err) time.Sleep(1 * time.Second) // try a frontend RefreshState that should timeout as the tablet is busy running the other one - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RefreshState", "--", primaryTablet.Alias, "--wait-time", "2s") + err = clusterInstance.VtctlclientProcess.ExecuteCommand("RefreshState", primaryTablet.Alias, "--wait_timeout", "2s") assert.Error(t, err, "timeout as tablet is in Sleep") } @@ -207,14 +206,14 @@ func TestShardReplicationFix(t *testing.T) { assertNodeCount(t, result, int(3)) // Manually add a bogus entry to the replication graph, and check it is removed by ShardReplicationFix - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ShardReplicationAdd", keyspaceShard, fmt.Sprintf("%s-9000", cell)) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ShardReplicationAdd", keyspaceShard, fmt.Sprintf("%s-9000", cell)) require.Nil(t, err, "error should be Nil") result, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShardReplication", cell, keyspaceShard) require.Nil(t, err, "error should be Nil") assertNodeCount(t, result, int(4)) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ShardReplicationFix", cell, keyspaceShard) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ShardReplicationFix", cell, keyspaceShard) require.Nil(t, err, "error should be Nil") result, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShardReplication", cell, keyspaceShard) require.Nil(t, err, "error should be Nil") @@ -224,7 +223,7 @@ func TestShardReplicationFix(t *testing.T) { func TestGetSchema(t *testing.T) { defer cluster.PanicHandler(t) - res, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetSchema", "--", + res, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("GetSchema", "--include-views", "--tables", "t1,v1", fmt.Sprintf("%s-%d", clusterInstance.Cell, primaryTablet.TabletUID)) require.Nil(t, err) diff --git a/go/test/endtoend/tabletmanager/custom_rule_topo_test.go b/go/test/endtoend/tabletmanager/custom_rule_topo_test.go index aa09a99e0fe..0c6e056af36 100644 --- a/go/test/endtoend/tabletmanager/custom_rule_topo_test.go +++ b/go/test/endtoend/tabletmanager/custom_rule_topo_test.go @@ -74,7 +74,7 @@ func TestTopoCustomRule(t *testing.T) { err = clusterInstance.StartVttablet(rTablet, false, "SERVING", false, cell, keyspaceName, hostname, shardName) require.Nil(t, err, "error should be Nil") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Validate") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("Validate") require.Nil(t, err, "error should be Nil") // And wait until the query is working. diff --git a/go/test/endtoend/tabletmanager/primary/tablet_test.go b/go/test/endtoend/tabletmanager/primary/tablet_test.go index f6255b1f71a..297e5540fac 100644 --- a/go/test/endtoend/tabletmanager/primary/tablet_test.go +++ b/go/test/endtoend/tabletmanager/primary/tablet_test.go @@ -26,7 +26,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/json2" "vitess.io/vitess/go/test/endtoend/cluster" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -121,16 +120,16 @@ func TestRepeatedInitShardPrimary(t *testing.T) { // Test that using InitShardPrimary can go back and forth between 2 hosts. // Make replica tablet as primary - err := clusterInstance.VtctlclientProcess.InitShardPrimary(keyspaceName, shardName, cell, replicaTablet.TabletUID) + err := clusterInstance.VtctldClientProcess.InitShardPrimary(keyspaceName, shardName, cell, replicaTablet.TabletUID) require.NoError(t, err) // Run health check on both, make sure they are both healthy. // Also make sure the types are correct. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", primaryTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", primaryTablet.Alias) require.NoError(t, err) checkHealth(t, primaryTablet.HTTPPort, false) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", replicaTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", replicaTablet.Alias) require.NoError(t, err) checkHealth(t, replicaTablet.HTTPPort, false) @@ -138,16 +137,16 @@ func TestRepeatedInitShardPrimary(t *testing.T) { checkTabletType(t, replicaTablet.Alias, "PRIMARY") // Come back to the original tablet. - err = clusterInstance.VtctlclientProcess.InitShardPrimary(keyspaceName, shardName, cell, primaryTablet.TabletUID) + err = clusterInstance.VtctldClientProcess.InitShardPrimary(keyspaceName, shardName, cell, primaryTablet.TabletUID) require.NoError(t, err) // Run health check on both, make sure they are both healthy. // Also make sure the types are correct. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", primaryTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", primaryTablet.Alias) require.NoError(t, err) checkHealth(t, primaryTablet.HTTPPort, false) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", replicaTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", replicaTablet.Alias) require.NoError(t, err) checkHealth(t, replicaTablet.HTTPPort, false) @@ -162,7 +161,7 @@ func TestPrimaryRestartSetsPTSTimestamp(t *testing.T) { // See StreamHealthResponse.primary_term_start_timestamp for details. // Make replica as primary - err := clusterInstance.VtctlclientProcess.InitShardPrimary(keyspaceName, shardName, cell, replicaTablet.TabletUID) + err := clusterInstance.VtctldClientProcess.InitShardPrimary(keyspaceName, shardName, cell, replicaTablet.TabletUID) require.NoError(t, err) err = replicaTablet.VttabletProcess.WaitForTabletStatus("SERVING") @@ -212,7 +211,7 @@ func TestPrimaryRestartSetsPTSTimestamp(t *testing.T) { streamHealthRes2.GetPrimaryTermStartTimestamp())) // Reset primary - err = clusterInstance.VtctlclientProcess.InitShardPrimary(keyspaceName, shardName, cell, primaryTablet.TabletUID) + err = clusterInstance.VtctldClientProcess.InitShardPrimary(keyspaceName, shardName, cell, primaryTablet.TabletUID) require.NoError(t, err) err = primaryTablet.VttabletProcess.WaitForTabletStatus("SERVING") require.NoError(t, err) @@ -232,11 +231,7 @@ func checkHealth(t *testing.T, port int, shouldError bool) { } func checkTabletType(t *testing.T, tabletAlias string, typeWant string) { - result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", tabletAlias) - require.NoError(t, err) - - var tablet topodatapb.Tablet - err = json2.Unmarshal([]byte(result), &tablet) + tablet, err := clusterInstance.VtctldClientProcess.GetTablet(tabletAlias) require.NoError(t, err) actualType := tablet.GetType() diff --git a/go/test/endtoend/tabletmanager/tablet_health_test.go b/go/test/endtoend/tabletmanager/tablet_health_test.go index 7dc4bcd97d2..c8eb43b682f 100644 --- a/go/test/endtoend/tabletmanager/tablet_health_test.go +++ b/go/test/endtoend/tabletmanager/tablet_health_test.go @@ -30,7 +30,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/json2" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" @@ -85,7 +84,7 @@ func TestTabletReshuffle(t *testing.T) { require.NoError(t, err) assertExcludeFields(t, string(result)) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", rTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("Backup", rTablet.Alias) assert.Error(t, err, "cannot perform backup without my.cnf") killTablets(rTablet) @@ -114,7 +113,7 @@ func TestHealthCheck(t *testing.T) { require.NoError(t, err) defer conn.Close() - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) require.NoError(t, err) checkHealth(t, rTablet.HTTPPort, false) @@ -123,9 +122,9 @@ func TestHealthCheck(t *testing.T) { utils.Exec(t, conn, "stop slave") // stop replication, make sure we don't go unhealthy. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", rTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("StopReplication", rTablet.Alias) require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) require.NoError(t, err) // make sure the health stream is updated @@ -136,9 +135,9 @@ func TestHealthCheck(t *testing.T) { } // then restart replication, make sure we stay healthy - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", rTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("StartReplication", rTablet.Alias) require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) require.NoError(t, err) checkHealth(t, rTablet.HTTPPort, false) @@ -173,16 +172,16 @@ func TestHealthCheck(t *testing.T) { // On a MySQL restart, it comes up as a read-only tablet (check default.cnf file). // We have to explicitly set it to read-write otherwise heartbeat writer is unable // to write the heartbeats - err = clusterInstance.VtctlclientProcess.ExecuteCommand("SetReadWrite", primaryTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("SetWritable", primaryTablet.Alias, "true") require.NoError(t, err) // explicitly start replication on all of the replicas to avoid any test flakiness as they were all // replicating from the primary instance - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", rTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("StartReplication", rTablet.Alias) require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", replicaTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("StartReplication", replicaTablet.Alias) require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", rdonlyTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("StartReplication", rdonlyTablet.Alias) require.NoError(t, err) time.Sleep(tabletHealthcheckRefreshInterval) @@ -348,11 +347,7 @@ func checkHealth(t *testing.T, port int, shouldError bool) { } func checkTabletType(t *testing.T, tabletAlias string, typeWant string) { - result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", tabletAlias) - require.NoError(t, err) - - var tablet topodatapb.Tablet - err = json2.Unmarshal([]byte(result), &tablet) + tablet, err := clusterInstance.VtctldClientProcess.GetTablet(tabletAlias) require.NoError(t, err) actualType := tablet.GetType() @@ -398,16 +393,16 @@ func TestHealthCheckDrainedStateDoesNotShutdownQueryService(t *testing.T) { // Change from rdonly to drained and stop replication. The tablet will stay // healthy, and the query service is still running. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "drained") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "drained") require.NoError(t, err) // Trying to drain the same tablet again, should error - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "drained") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "drained") assert.Error(t, err, "already drained") - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", rdonlyTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("StopReplication", rdonlyTablet.Alias) require.NoError(t, err) // Trigger healthcheck explicitly to avoid waiting for the next interval. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rdonlyTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", rdonlyTablet.Alias) require.NoError(t, err) checkTabletType(t, rdonlyTablet.Alias, "DRAINED") @@ -417,11 +412,11 @@ func TestHealthCheckDrainedStateDoesNotShutdownQueryService(t *testing.T) { require.NoError(t, err) // Restart replication. Tablet will become healthy again. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "rdonly") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "rdonly") require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", rdonlyTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("StartReplication", rdonlyTablet.Alias) require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rdonlyTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", rdonlyTablet.Alias) require.NoError(t, err) checkHealth(t, rdonlyTablet.HTTPPort, false) } @@ -434,7 +429,7 @@ func killTablets(tablets ...*cluster.Vttablet) { defer wg.Done() _ = tablet.VttabletProcess.TearDown() _ = tablet.MysqlctlProcess.Stop() - _ = clusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", tablet.Alias) + _ = clusterInstance.VtctldClientProcess.ExecuteCommand("DeleteTablets", tablet.Alias) }(tablet) } wg.Wait() diff --git a/go/test/endtoend/tabletmanager/tablet_test.go b/go/test/endtoend/tabletmanager/tablet_test.go index 6212b4a418b..830502268d1 100644 --- a/go/test/endtoend/tabletmanager/tablet_test.go +++ b/go/test/endtoend/tabletmanager/tablet_test.go @@ -47,7 +47,7 @@ func TestEnsureDB(t *testing.T) { require.NoError(t, err) // Make it the primary. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("TabletExternallyReparented", tablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("TabletExternallyReparented", tablet.Alias) require.EqualError(t, err, "exit status 1") // It is still NOT_SERVING because the db is read-only. @@ -56,8 +56,8 @@ func TestEnsureDB(t *testing.T) { assert.Contains(t, status, "read-only") // Switch to read-write and verify that we go serving. - // Note: for TabletExternallyReparented, we expect SetReadWrite to be called by the user - err = clusterInstance.VtctlclientProcess.ExecuteCommand("SetReadWrite", tablet.Alias) + // Note: for TabletExternallyReparented, we expect SetWritable to be called by the user + err = clusterInstance.VtctldClientProcess.ExecuteCommand("SetWritable", tablet.Alias, "true") require.NoError(t, err) err = tablet.VttabletProcess.WaitForTabletStatus("SERVING") require.NoError(t, err) diff --git a/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go b/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go index 7c0f05bdcc2..9824f28ae2b 100644 --- a/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go +++ b/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go @@ -370,7 +370,7 @@ func TestLag(t *testing.T) { defer clusterInstance.EnableVTOrcRecoveries(t) t.Run("stopping replication", func(t *testing.T) { - err := clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", replicaTablet.Alias) + err := clusterInstance.VtctldClientProcess.ExecuteCommand("StopReplication", replicaTablet.Alias) assert.NoError(t, err) }) t.Run("accumulating lag, expecting throttler push back", func(t *testing.T) { @@ -415,7 +415,7 @@ func TestLag(t *testing.T) { }) t.Run("starting replication", func(t *testing.T) { - err := clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", replicaTablet.Alias) + err := clusterInstance.VtctldClientProcess.ExecuteCommand("StartReplication", replicaTablet.Alias) assert.NoError(t, err) }) t.Run("expecting replication to catch up and throttler check to return OK", func(t *testing.T) { @@ -439,7 +439,7 @@ func TestLag(t *testing.T) { func TestNoReplicas(t *testing.T) { defer cluster.PanicHandler(t) t.Run("changing replica to RDONLY", func(t *testing.T) { - err := clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", replicaTablet.Alias, "RDONLY") + err := clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", replicaTablet.Alias, "RDONLY") assert.NoError(t, err) // This makes no REPLICA servers available. We expect something like: @@ -447,7 +447,7 @@ func TestNoReplicas(t *testing.T) { waitForThrottleCheckStatus(t, primaryTablet, http.StatusOK) }) t.Run("restoring to REPLICA", func(t *testing.T) { - err := clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", replicaTablet.Alias, "REPLICA") + err := clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", replicaTablet.Alias, "REPLICA") assert.NoError(t, err) waitForThrottleCheckStatus(t, primaryTablet, http.StatusOK) diff --git a/go/test/endtoend/topoconncache/main_test.go b/go/test/endtoend/topoconncache/main_test.go index 4c17481ec84..26eb3918a0b 100644 --- a/go/test/endtoend/topoconncache/main_test.go +++ b/go/test/endtoend/topoconncache/main_test.go @@ -193,14 +193,14 @@ func TestMain(m *testing.M) { return 1, err } } - if err := clusterInstance.VtctlclientProcess.InitializeShard(keyspaceName, shard1.Name, shard1Primary.Cell, shard1Primary.TabletUID); err != nil { + if err := clusterInstance.VtctldClientProcess.InitializeShard(keyspaceName, shard1.Name, shard1Primary.Cell, shard1Primary.TabletUID); err != nil { return 1, err } // run a health check on source replica so it responds to discovery // (for binlog players) and on the source rdonlys (for workers) for _, tablet := range []string{shard1Replica.Alias, shard1Rdonly.Alias} { - if err := clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", tablet); err != nil { + if err := clusterInstance.VtctldClientProcess.ExecuteCommand("RunHealthCheck", tablet); err != nil { return 1, err } } @@ -211,7 +211,7 @@ func TestMain(m *testing.M) { } } - if err := clusterInstance.VtctlclientProcess.InitializeShard(keyspaceName, shard2.Name, shard2Primary.Cell, shard2Primary.TabletUID); err != nil { + if err := clusterInstance.VtctldClientProcess.InitializeShard(keyspaceName, shard2.Name, shard2Primary.Cell, shard2Primary.TabletUID); err != nil { return 1, err } @@ -219,14 +219,14 @@ func TestMain(m *testing.M) { return 1, err } - if err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, fmt.Sprintf(sqlSchema, tableName)); err != nil { + if err := clusterInstance.VtctldClientProcess.ApplySchema(keyspaceName, fmt.Sprintf(sqlSchema, tableName)); err != nil { return 1, err } - if err := clusterInstance.VtctlclientProcess.ApplyVSchema(keyspaceName, fmt.Sprintf(vSchema, tableName)); err != nil { + if err := clusterInstance.VtctldClientProcess.ApplyVSchema(keyspaceName, fmt.Sprintf(vSchema, tableName)); err != nil { return 1, err } - _ = clusterInstance.VtctlclientProcess.ExecuteCommand("RebuildKeyspaceGraph", keyspaceName) + _ = clusterInstance.VtctldClientProcess.ExecuteCommand("RebuildKeyspaceGraph", keyspaceName) return m.Run(), nil }() diff --git a/go/test/endtoend/topoconncache/topo_conn_cache_test.go b/go/test/endtoend/topoconncache/topo_conn_cache_test.go index 504ca218047..4ffcc309e29 100644 --- a/go/test/endtoend/topoconncache/topo_conn_cache_test.go +++ b/go/test/endtoend/topoconncache/topo_conn_cache_test.go @@ -74,7 +74,7 @@ func deleteCell(t *testing.T) { deleteTablet(t, shard2Rdonly) // Delete cell2 info from topo - res, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("DeleteCellInfo", "--", "--force", cell2) + res, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("DeleteCellInfo", "--force", cell2) t.Log(res) require.NoError(t, err) @@ -111,7 +111,7 @@ func deleteTablet(t *testing.T, tablet *cluster.Vttablet) { }(tablet) wg.Wait() - err := clusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", tablet.Alias) + err := clusterInstance.VtctldClientProcess.ExecuteCommand("DeleteTablets", tablet.Alias) require.NoError(t, err) } diff --git a/go/test/endtoend/vault/vault_test.go b/go/test/endtoend/vault/vault_test.go index 684a374707d..f8e19c07a0c 100644 --- a/go/test/endtoend/vault/vault_test.go +++ b/go/test/endtoend/vault/vault_test.go @@ -283,7 +283,7 @@ func initializeClusterLate(t *testing.T) { tablet.MysqlctlProcess.ExtraArgs = append(tablet.MysqlctlProcess.ExtraArgs, mysqlctlArg...) } - err = clusterInstance.VtctlclientProcess.InitShardPrimary(keyspaceName, shard.Name, cell, primary.TabletUID) + err = clusterInstance.VtctldClientProcess.InitShardPrimary(keyspaceName, shard.Name, cell, primary.TabletUID) require.NoError(t, err) err = clusterInstance.StartVTOrc(keyspaceName) diff --git a/go/test/endtoend/versionupgrade/upgrade_test.go b/go/test/endtoend/versionupgrade/upgrade_test.go index 87f7f9e8675..181b5dfc9ad 100644 --- a/go/test/endtoend/versionupgrade/upgrade_test.go +++ b/go/test/endtoend/versionupgrade/upgrade_test.go @@ -148,7 +148,7 @@ func TestDeploySchema(t *testing.T) { { sqlQuery := fmt.Sprintf(createTable, tableName) - result, err := clusterInstance.VtctlclientProcess.ApplySchemaWithOutput(keyspaceName, sqlQuery, cluster.VtctlClientParams{DDLStrategy: ""}) + result, err := clusterInstance.VtctldClientProcess.ApplySchemaWithOutput(keyspaceName, sqlQuery, cluster.ApplySchemaParams{DDLStrategy: ""}) require.Nil(t, err, result) } for i := range clusterInstance.Keyspaces[0].Shards { diff --git a/go/test/endtoend/vtgate/createdb_plugin/main_test.go b/go/test/endtoend/vtgate/createdb_plugin/main_test.go index e712fee7b36..5bfec3890b5 100644 --- a/go/test/endtoend/vtgate/createdb_plugin/main_test.go +++ b/go/test/endtoend/vtgate/createdb_plugin/main_test.go @@ -164,8 +164,8 @@ func shutdown(t *testing.T, ksName string) { } require.NoError(t, - clusterInstance.VtctlclientProcess.ExecuteCommand("DeleteKeyspace", "--", "--recursive", ksName)) + clusterInstance.VtctldClientProcess.ExecuteCommand("DeleteKeyspace", "--recursive", ksName)) require.NoError(t, - clusterInstance.VtctlclientProcess.ExecuteCommand("RebuildVSchemaGraph")) + clusterInstance.VtctldClientProcess.ExecuteCommand("RebuildVSchemaGraph")) } diff --git a/go/test/endtoend/vtgate/foreignkey/main_test.go b/go/test/endtoend/vtgate/foreignkey/main_test.go index 483c1d05e80..63590df9247 100644 --- a/go/test/endtoend/vtgate/foreignkey/main_test.go +++ b/go/test/endtoend/vtgate/foreignkey/main_test.go @@ -139,7 +139,7 @@ func TestMain(m *testing.M) { return 1 } - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RebuildVSchemaGraph") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RebuildVSchemaGraph") if err != nil { return 1 } diff --git a/go/test/endtoend/vtgate/foreignkey/stress/fk_stress_test.go b/go/test/endtoend/vtgate/foreignkey/stress/fk_stress_test.go index 60eff73b820..341db836fd8 100644 --- a/go/test/endtoend/vtgate/foreignkey/stress/fk_stress_test.go +++ b/go/test/endtoend/vtgate/foreignkey/stress/fk_stress_test.go @@ -597,7 +597,7 @@ func ExecuteFKTest(t *testing.T, tcase *testCase) { artifacts := textutil.SplitDelimitedList(row.AsString("artifacts", "")) for _, artifact := range artifacts { t.Run(artifact, func(t *testing.T) { - err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, "drop table if exists "+artifact) + err := clusterInstance.VtctldClientProcess.ApplySchema(keyspaceName, "drop table if exists "+artifact) require.NoError(t, err) }) } @@ -781,7 +781,7 @@ func createInitialSchema(t *testing.T, tcase *testCase) { t.Run("dropping tables", func(t *testing.T) { for _, tableName := range reverseTableNames { - err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, "drop table if exists "+tableName) + err := clusterInstance.VtctldClientProcess.ApplySchema(keyspaceName, "drop table if exists "+tableName) require.NoError(t, err) } }) @@ -808,7 +808,7 @@ func createInitialSchema(t *testing.T, tcase *testCase) { } b.WriteString(";") } - err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, b.String()) + err := clusterInstance.VtctldClientProcess.ApplySchema(keyspaceName, b.String()) require.NoError(t, err) }) if tcase.preStatement != "" { @@ -862,7 +862,7 @@ func testOnlineDDLStatement(t *testing.T, alterStatement string, ddlStrategy str } } else { var err error - uuid, err = clusterInstance.VtctlclientProcess.ApplySchemaWithOutput(keyspaceName, alterStatement, cluster.VtctlClientParams{DDLStrategy: ddlStrategy}) + uuid, err = clusterInstance.VtctldClientProcess.ApplySchemaWithOutput(keyspaceName, alterStatement, cluster.ApplySchemaParams{DDLStrategy: ddlStrategy}) assert.NoError(t, err) } uuid = strings.TrimSpace(uuid) diff --git a/go/test/endtoend/vtgate/gen4/main_test.go b/go/test/endtoend/vtgate/gen4/main_test.go index 378b2d2969e..4c94e8e2ec8 100644 --- a/go/test/endtoend/vtgate/gen4/main_test.go +++ b/go/test/endtoend/vtgate/gen4/main_test.go @@ -102,12 +102,12 @@ func TestMain(m *testing.M) { } // apply routing rules - err = clusterInstance.VtctlclientProcess.ApplyRoutingRules(routingRules) + err = clusterInstance.VtctldClientProcess.ApplyRoutingRules(routingRules) if err != nil { return 1 } - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RebuildVSchemaGraph") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RebuildVSchemaGraph") if err != nil { return 1 } diff --git a/go/test/endtoend/vtgate/main_test.go b/go/test/endtoend/vtgate/main_test.go index 12abcf4dd01..b276508f269 100644 --- a/go/test/endtoend/vtgate/main_test.go +++ b/go/test/endtoend/vtgate/main_test.go @@ -79,12 +79,12 @@ func TestMain(m *testing.M) { return 1 } - err = clusterInstance.VtctlclientProcess.ApplyRoutingRules(routingRules) + err = clusterInstance.VtctldClientProcess.ApplyRoutingRules(routingRules) if err != nil { return 1 } - _, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("RebuildVSchemaGraph") + _, err = clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("RebuildVSchemaGraph") if err != nil { return 1 } diff --git a/go/test/endtoend/vtgate/queries/informationschema/main_test.go b/go/test/endtoend/vtgate/queries/informationschema/main_test.go index 06c5b188d18..3696617281e 100644 --- a/go/test/endtoend/vtgate/queries/informationschema/main_test.go +++ b/go/test/endtoend/vtgate/queries/informationschema/main_test.go @@ -78,12 +78,12 @@ func TestMain(m *testing.M) { return 1 } - err = clusterInstance.VtctlclientProcess.ApplyRoutingRules(routingRules) + err = clusterInstance.VtctldClientProcess.ApplyRoutingRules(routingRules) if err != nil { return 1 } - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RebuildVSchemaGraph") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RebuildVSchemaGraph") if err != nil { return 1 } diff --git a/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go b/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go index c385941502a..9486dc194ff 100644 --- a/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go +++ b/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go @@ -74,7 +74,7 @@ func TestMain(m *testing.M) { return 1 } - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RebuildVSchemaGraph") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("RebuildVSchemaGraph") if err != nil { return 1 } diff --git a/go/test/endtoend/vtgate/reservedconn/reconnect1/main_test.go b/go/test/endtoend/vtgate/reservedconn/reconnect1/main_test.go index 11325a0f2f8..491ce6bc6ab 100644 --- a/go/test/endtoend/vtgate/reservedconn/reconnect1/main_test.go +++ b/go/test/endtoend/vtgate/reservedconn/reconnect1/main_test.go @@ -133,7 +133,7 @@ func TestServingChange(t *testing.T) { // changing rdonly tablet to spare (non serving). rdonlyTablet := clusterInstance.Keyspaces[0].Shards[0].Rdonly() - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "replica") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "replica") require.NoError(t, err) rdonlyTablet.Type = "replica" @@ -143,12 +143,12 @@ func TestServingChange(t *testing.T) { // changing replica tablet to rdonly to make rdonly available for serving. replicaTablet := clusterInstance.Keyspaces[0].Shards[0].Replica() - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", replicaTablet.Alias, "rdonly") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", replicaTablet.Alias, "rdonly") require.NoError(t, err) replicaTablet.Type = "rdonly" // to see/make the new rdonly available - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Ping", replicaTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("PingTablet", replicaTablet.Alias) require.NoError(t, err) // this should pass now as there is rdonly present @@ -174,7 +174,7 @@ func TestServingChangeStreaming(t *testing.T) { // changing rdonly tablet to spare (non serving). rdonlyTablet := clusterInstance.Keyspaces[0].Shards[0].Rdonly() - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "replica") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", rdonlyTablet.Alias, "replica") require.NoError(t, err) rdonlyTablet.Type = "replica" @@ -192,12 +192,12 @@ func TestServingChangeStreaming(t *testing.T) { // changing replica tablet to rdonly to make rdonly available for serving. replicaTablet := clusterInstance.Keyspaces[0].Shards[0].Replica() - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", replicaTablet.Alias, "rdonly") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ChangeTabletType", replicaTablet.Alias, "rdonly") require.NoError(t, err) replicaTablet.Type = "rdonly" // to see/make the new rdonly available - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Ping", replicaTablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("PingTablet", replicaTablet.Alias) require.NoError(t, err) // this should pass now as there is rdonly present diff --git a/go/test/endtoend/vtgate/reservedconn/reconnect2/main_test.go b/go/test/endtoend/vtgate/reservedconn/reconnect2/main_test.go index b66bb15dbd5..a448574c282 100644 --- a/go/test/endtoend/vtgate/reservedconn/reconnect2/main_test.go +++ b/go/test/endtoend/vtgate/reservedconn/reconnect2/main_test.go @@ -129,7 +129,7 @@ func TestTabletChange(t *testing.T) { utils.Exec(t, conn, "select * from test") // Change Primary - err = clusterInstance.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", "--", "--keyspace_shard", fmt.Sprintf("%s/%s", keyspaceName, "-80")) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("PlannedReparentShard", fmt.Sprintf("%s/%s", keyspaceName, "-80")) require.NoError(t, err) // this should pass as there is a new primary tablet and is serving. @@ -150,7 +150,7 @@ func TestTabletChangeStreaming(t *testing.T) { utils.Exec(t, conn, "select * from test") // Change Primary - err = clusterInstance.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", "--", "--keyspace_shard", fmt.Sprintf("%s/%s", keyspaceName, "-80")) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("PlannedReparentShard", fmt.Sprintf("%s/%s", keyspaceName, "-80")) require.NoError(t, err) // this should pass as there is a new primary tablet and is serving. diff --git a/go/test/endtoend/vtgate/reservedconn/reconnect3/main_test.go b/go/test/endtoend/vtgate/reservedconn/reconnect3/main_test.go index 25af85acc00..677c24666b2 100644 --- a/go/test/endtoend/vtgate/reservedconn/reconnect3/main_test.go +++ b/go/test/endtoend/vtgate/reservedconn/reconnect3/main_test.go @@ -102,7 +102,7 @@ func TestMysqlDownServingChange(t *testing.T) { require.NoError(t, primaryTablet.MysqlctlProcess.Stop()) require.NoError(t, - clusterInstance.VtctlclientProcess.ExecuteCommand("EmergencyReparentShard", "--", "--keyspace_shard", "ks/0")) + clusterInstance.VtctldClientProcess.ExecuteCommand("EmergencyReparentShard", "ks/0")) // This should work without any error. _ = utils.Exec(t, conn, "select /*vt+ PLANNER=gen4 */ * from test") diff --git a/go/test/endtoend/vtgate/reservedconn/reconnect4/main_test.go b/go/test/endtoend/vtgate/reservedconn/reconnect4/main_test.go index 28367cd597a..1dc53a89506 100644 --- a/go/test/endtoend/vtgate/reservedconn/reconnect4/main_test.go +++ b/go/test/endtoend/vtgate/reservedconn/reconnect4/main_test.go @@ -104,7 +104,7 @@ func TestVttabletDownServingChange(t *testing.T) { // kill vttablet process _ = primaryTablet.VttabletProcess.TearDown() require.NoError(t, - clusterInstance.VtctlclientProcess.ExecuteCommand("EmergencyReparentShard", "--", "--keyspace_shard", "ks/0")) + clusterInstance.VtctldClientProcess.ExecuteCommand("EmergencyReparentShard", "ks/0")) // This should work without any error. _ = utils.Exec(t, conn, "select /*vt+ PLANNER=gen4 */ * from test") diff --git a/go/test/endtoend/vtgate/schema/schema_test.go b/go/test/endtoend/vtgate/schema/schema_test.go index 04d91d8d978..14b6c13034e 100644 --- a/go/test/endtoend/vtgate/schema/schema_test.go +++ b/go/test/endtoend/vtgate/schema/schema_test.go @@ -120,7 +120,7 @@ func testWithInitialSchema(t *testing.T) { var sqlQuery = "" // nolint for i := 0; i < totalTableCount; i++ { sqlQuery = fmt.Sprintf(createTable, fmt.Sprintf("vt_select_test_%02d", i)) - err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, sqlQuery) + err := clusterInstance.VtctldClientProcess.ApplySchema(keyspaceName, sqlQuery) require.Nil(t, err) } @@ -135,7 +135,7 @@ func testWithInitialSchema(t *testing.T) { // testWithAlterSchema if we alter schema and then apply, the resultant schema should match across shards func testWithAlterSchema(t *testing.T) { sqlQuery := fmt.Sprintf(alterTable, fmt.Sprintf("vt_select_test_%02d", 3), "msg") - err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, sqlQuery) + err := clusterInstance.VtctldClientProcess.ApplySchema(keyspaceName, sqlQuery) require.Nil(t, err) matchSchema(t, clusterInstance.Keyspaces[0].Shards[0].Vttablets[0].VttabletProcess.TabletPath, clusterInstance.Keyspaces[0].Shards[1].Vttablets[0].VttabletProcess.TabletPath) } @@ -143,7 +143,7 @@ func testWithAlterSchema(t *testing.T) { // testWithAlterDatabase tests that ALTER DATABASE is accepted by the validator. func testWithAlterDatabase(t *testing.T) { sql := "create database alter_database_test; alter database alter_database_test default character set = utf8mb4; drop database alter_database_test" - err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, sql) + err := clusterInstance.VtctldClientProcess.ApplySchema(keyspaceName, sql) assert.NoError(t, err) } @@ -157,7 +157,7 @@ func testWithAlterDatabase(t *testing.T) { // See: https://github.com/vitessio/vitess/issues/1731#issuecomment-222914389 func testWithDropCreateSchema(t *testing.T) { dropCreateTable := fmt.Sprintf("DROP TABLE vt_select_test_%02d ;", 2) + fmt.Sprintf(createTable, fmt.Sprintf("vt_select_test_%02d", 2)) - err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, dropCreateTable) + err := clusterInstance.VtctldClientProcess.ApplySchema(keyspaceName, dropCreateTable) require.NoError(t, err) checkTables(t, totalTableCount) } @@ -186,10 +186,10 @@ func testWithAutoSchemaFromChangeDir(t *testing.T) { // matchSchema schema for supplied tablets should match func matchSchema(t *testing.T, firstTablet string, secondTablet string) { - firstShardSchema, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetSchema", firstTablet) + firstShardSchema, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("GetSchema", firstTablet) require.Nil(t, err) - secondShardSchema, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetSchema", secondTablet) + secondShardSchema, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("GetSchema", secondTablet) require.Nil(t, err) assert.Equal(t, firstShardSchema, secondShardSchema) @@ -203,12 +203,12 @@ func matchSchema(t *testing.T, firstTablet string, secondTablet string) { // is the MySQL behavior the user expects. func testDropNonExistentTables(t *testing.T) { dropNonExistentTable := "DROP TABLE nonexistent_table;" - output, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "--", "--sql", dropNonExistentTable, keyspaceName) + output, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("ApplySchema", "--sql", dropNonExistentTable, keyspaceName) require.Error(t, err) assert.True(t, strings.Contains(output, "Unknown table")) dropIfExists := "DROP TABLE IF EXISTS nonexistent_table;" - err = clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, dropIfExists) + err = clusterInstance.VtctldClientProcess.ApplySchema(keyspaceName, dropIfExists) require.Nil(t, err) checkTables(t, totalTableCount) @@ -219,7 +219,7 @@ func testDropNonExistentTables(t *testing.T) { func testCreateInvalidView(t *testing.T) { for _, ddlStrategy := range []string{"direct", "direct -allow-zero-in-date"} { createInvalidView := "CREATE OR REPLACE VIEW invalid_view AS SELECT * FROM nonexistent_table;" - output, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "--", "--ddl_strategy", ddlStrategy, "--sql", createInvalidView, keyspaceName) + output, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("ApplySchema", "--ddl-strategy", ddlStrategy, "--sql", createInvalidView, keyspaceName) require.Error(t, err) assert.Contains(t, output, "doesn't exist (errno 1146)") } @@ -228,25 +228,25 @@ func testCreateInvalidView(t *testing.T) { func testApplySchemaBatch(t *testing.T) { { sqls := "create table batch1(id int primary key);create table batch2(id int primary key);create table batch3(id int primary key);create table batch4(id int primary key);create table batch5(id int primary key);" - _, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "--", "--sql", sqls, "--batch_size", "2", keyspaceName) + _, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("ApplySchema", "--sql", sqls, "--batch-size", "2", keyspaceName) require.NoError(t, err) checkTables(t, totalTableCount+5) } { sqls := "drop table batch1; drop table batch2; drop table batch3; drop table batch4; drop table batch5" - _, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "--", "--sql", sqls, keyspaceName) + _, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("ApplySchema", "--sql", sqls, keyspaceName) require.NoError(t, err) checkTables(t, totalTableCount) } { sqls := "create table batch1(id int primary key);create table batch2(id int primary key);create table batch3(id int primary key);create table batch4(id int primary key);create table batch5(id int primary key);" - _, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "--", "--ddl_strategy", "direct --allow-zero-in-date", "--sql", sqls, "--batch_size", "2", keyspaceName) + _, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("ApplySchema", "--ddl-strategy", "direct --allow-zero-in-date", "--sql", sqls, "--batch-size", "2", keyspaceName) require.NoError(t, err) checkTables(t, totalTableCount+5) } { sqls := "drop table batch1; drop table batch2; drop table batch3; drop table batch4; drop table batch5" - _, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "--", "--sql", sqls, keyspaceName) + _, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("ApplySchema", "--sql", sqls, keyspaceName) require.NoError(t, err) checkTables(t, totalTableCount) } @@ -291,7 +291,7 @@ func testCopySchemaShardWithDifferentDB(t *testing.T, shard int) { source := fmt.Sprintf("%s/0", keyspaceName) tabletAlias := clusterInstance.Keyspaces[0].Shards[shard].Vttablets[0].VttabletProcess.TabletPath - schema, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetSchema", tabletAlias) + schema, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("GetSchema", tabletAlias) require.Nil(t, err) resultMap := make(map[string]any) @@ -305,7 +305,7 @@ func testCopySchemaShardWithDifferentDB(t *testing.T, shard int) { // (The different charset won't be corrected on the destination shard // because we use "CREATE DATABASE IF NOT EXISTS" and this doesn't fail if // there are differences in the options e.g. the character set.) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ExecuteFetchAsDba", "--", "--json", tabletAlias, "ALTER DATABASE vt_ks CHARACTER SET latin1") + err = clusterInstance.VtctldClientProcess.ExecuteCommand("ExecuteFetchAsDBA", "--json", tabletAlias, "ALTER DATABASE vt_ks CHARACTER SET latin1") require.Nil(t, err) output, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("CopySchemaShard", source, fmt.Sprintf("%s/%d", keyspaceName, shard)) diff --git a/go/test/endtoend/vtgate/schematracker/sharded_prs/st_sharded_test.go b/go/test/endtoend/vtgate/schematracker/sharded_prs/st_sharded_test.go index 6ff8e69bb52..09bd97eb9fe 100644 --- a/go/test/endtoend/vtgate/schematracker/sharded_prs/st_sharded_test.go +++ b/go/test/endtoend/vtgate/schematracker/sharded_prs/st_sharded_test.go @@ -181,7 +181,7 @@ func TestMain(m *testing.M) { // This is supposed to change the primary tablet in the shards, meaning that a different tablet // will be responsible for sending schema tracking updates. for _, shard := range clusterInstance.Keyspaces[0].Shards { - err := clusterInstance.VtctlclientProcess.InitializeShard(KeyspaceName, shard.Name, Cell, shard.Vttablets[1].TabletUID) + err := clusterInstance.VtctldClientProcess.InitializeShard(KeyspaceName, shard.Name, Cell, shard.Vttablets[1].TabletUID) if err != nil { fmt.Println(err) return 1 diff --git a/go/test/endtoend/vtgate/tablet_healthcheck/reparent_test.go b/go/test/endtoend/vtgate/tablet_healthcheck/reparent_test.go index dbc46bdda77..d6357ce8f2a 100644 --- a/go/test/endtoend/vtgate/tablet_healthcheck/reparent_test.go +++ b/go/test/endtoend/vtgate/tablet_healthcheck/reparent_test.go @@ -145,11 +145,11 @@ func TestHealthCheckExternallyReparentNewTablet(t *testing.T) { tablet := addTablet(t, reparentTabletUID, reparentTabletType) // promote the new tablet to the primary - err = clusterInstance.VtctlclientProcess.ExecuteCommand("TabletExternallyReparented", tablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("TabletExternallyReparented", tablet.Alias) require.NoError(t, err) // update the new primary tablet to be read-write - err = clusterInstance.VtctlclientProcess.ExecuteCommand("SetReadWrite", tablet.Alias) + err = clusterInstance.VtctldClientProcess.ExecuteCommand("SetWritable", tablet.Alias, "true") require.NoError(t, err) // wait for the vtgate to finish updating the new primary tablet @@ -236,7 +236,7 @@ func deleteTablet(t *testing.T, tablet *cluster.Vttablet) { }(tablet) wg.Wait() - err := clusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", tablet.Alias) + err := clusterInstance.VtctldClientProcess.ExecuteCommand("DeleteTablets", tablet.Alias) require.NoError(t, err) t.Logf("Deleted tablet: %s", tablet.Alias) diff --git a/go/test/endtoend/vtgate/tablet_healthcheck_cache/correctness_test.go b/go/test/endtoend/vtgate/tablet_healthcheck_cache/correctness_test.go index 9386c307a12..50529d9fdf9 100644 --- a/go/test/endtoend/vtgate/tablet_healthcheck_cache/correctness_test.go +++ b/go/test/endtoend/vtgate/tablet_healthcheck_cache/correctness_test.go @@ -234,7 +234,7 @@ func deleteTablet(t *testing.T, tablet *cluster.Vttablet) { }(tablet) wg.Wait() - err := clusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", tablet.Alias) + err := clusterInstance.VtctldClientProcess.ExecuteCommand("DeleteTablets", tablet.Alias) require.Nil(t, err) t.Logf("Deleted tablet: %s", tablet.Alias) diff --git a/go/test/endtoend/vtorc/api/api_test.go b/go/test/endtoend/vtorc/api/api_test.go index 7dd5c50eefa..7b277fd7a0f 100644 --- a/go/test/endtoend/vtorc/api/api_test.go +++ b/go/test/endtoend/vtorc/api/api_test.go @@ -106,7 +106,7 @@ func TestAPIEndpoints(t *testing.T) { t.Run("Replication Analysis API", func(t *testing.T) { // use vtctlclient to stop replication - _, err := clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("StopReplication", replica.Alias) + _, err := clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("StopReplication", replica.Alias) require.NoError(t, err) // We know VTOrc won't fix this since we disabled global recoveries! diff --git a/go/test/endtoend/vtorc/general/vtorc_test.go b/go/test/endtoend/vtorc/general/vtorc_test.go index f7deffe20b3..d79e2964f3e 100644 --- a/go/test/endtoend/vtorc/general/vtorc_test.go +++ b/go/test/endtoend/vtorc/general/vtorc_test.go @@ -163,7 +163,7 @@ func TestVTOrcRepairs(t *testing.T) { t.Run("StopReplication", func(t *testing.T) { // use vtctlclient to stop replication - _, err := clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("StopReplication", replica.Alias) + _, err := clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("StopReplication", replica.Alias) require.NoError(t, err) // check replication is setup correctly @@ -300,7 +300,7 @@ func TestRepairAfterTER(t *testing.T) { } // TER to other tablet - _, err = clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("TabletExternallyReparented", newPrimary.Alias) + _, err = clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("TabletExternallyReparented", newPrimary.Alias) require.NoError(t, err) utils.CheckReplication(t, clusterInfo, newPrimary, []*cluster.Vttablet{curPrimary}, 15*time.Second) @@ -404,11 +404,11 @@ func TestVTOrcWithPrs(t *testing.T) { // check that the replication is setup correctly before we failover utils.CheckReplication(t, clusterInfo, curPrimary, shard0.Vttablets, 10*time.Second) - output, err := clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput( - "PlannedReparentShard", "--", - "--keyspace_shard", fmt.Sprintf("%s/%s", keyspace.Name, shard0.Name), - "--wait_replicas_timeout", "31s", - "--new_primary", replica.Alias) + output, err := clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommandWithOutput( + "PlannedReparentShard", + fmt.Sprintf("%s/%s", keyspace.Name, shard0.Name), + "--wait-replicas-timeout", "31s", + "--new-primary", replica.Alias) require.NoError(t, err, "error in PlannedReparentShard output - %s", output) time.Sleep(40 * time.Second) diff --git a/go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go b/go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go index e226e8d13ae..d91dadddcb4 100644 --- a/go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go @@ -113,7 +113,7 @@ func TestDownPrimaryBeforeVTOrc(t *testing.T) { curPrimary := shard0.Vttablets[0] // Promote the first tablet as the primary - err := clusterInfo.ClusterInstance.VtctlclientProcess.InitializeShard(keyspace.Name, shard0.Name, clusterInfo.ClusterInstance.Cell, curPrimary.TabletUID) + err := clusterInfo.ClusterInstance.VtctldClientProcess.InitializeShard(keyspace.Name, shard0.Name, clusterInfo.ClusterInstance.Cell, curPrimary.TabletUID) require.NoError(t, err) // find the replica and rdonly tablets @@ -442,9 +442,9 @@ func TestLostRdonlyOnPrimaryFailure(t *testing.T) { utils.DisableGlobalRecoveries(t, clusterInfo.ClusterInstance.VTOrcProcesses[0]) // stop replication on the replica and rdonly. - err := clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", replica.Alias) + err := clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommand("StopReplication", replica.Alias) require.NoError(t, err) - err = clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", rdonly.Alias) + err = clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommand("StopReplication", rdonly.Alias) require.NoError(t, err) // check that aheadRdonly is able to replicate. We also want to add some queries to aheadRdonly which will not be there in replica and rdonly @@ -669,7 +669,7 @@ func TestDownPrimaryPromotionRuleWithLag(t *testing.T) { utils.DisableGlobalRecoveries(t, clusterInfo.ClusterInstance.VTOrcProcesses[0]) // stop replication on the crossCellReplica. - err := clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", crossCellReplica.Alias) + err := clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommand("StopReplication", crossCellReplica.Alias) require.NoError(t, err) // check that rdonly and replica are able to replicate. We also want to add some queries to replica which will not be there in crossCellReplica @@ -679,7 +679,7 @@ func TestDownPrimaryPromotionRuleWithLag(t *testing.T) { utils.ResetPrimaryLogs(t, curPrimary) // start replication back on the crossCellReplica. - err = clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", crossCellReplica.Alias) + err = clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommand("StartReplication", crossCellReplica.Alias) require.NoError(t, err) // enable recoveries back on vtorc so that it can repair @@ -750,7 +750,7 @@ func TestDownPrimaryPromotionRuleWithLagCrossCenter(t *testing.T) { utils.DisableGlobalRecoveries(t, clusterInfo.ClusterInstance.VTOrcProcesses[0]) // stop replication on the replica. - err := clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", replica.Alias) + err := clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommand("StopReplication", replica.Alias) require.NoError(t, err) // check that rdonly and crossCellReplica are able to replicate. We also want to add some queries to crossCenterReplica which will not be there in replica @@ -760,7 +760,7 @@ func TestDownPrimaryPromotionRuleWithLagCrossCenter(t *testing.T) { utils.ResetPrimaryLogs(t, curPrimary) // start replication back on the replica. - err = clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", replica.Alias) + err = clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommand("StartReplication", replica.Alias) require.NoError(t, err) // enable recoveries back on vtorc so that it can repair diff --git a/go/test/endtoend/vtorc/utils/utils.go b/go/test/endtoend/vtorc/utils/utils.go index 11294319658..dca2c7b1e26 100644 --- a/go/test/endtoend/vtorc/utils/utils.go +++ b/go/test/endtoend/vtorc/utils/utils.go @@ -33,7 +33,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/json2" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" @@ -206,7 +205,7 @@ func shutdownVttablets(clusterInfo *VTOrcClusterInfo) error { // Remove the tablet record for this tablet } // Ignoring error here because some tests delete tablets themselves. - _ = clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", vttablet.Alias) + _ = clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommand("DeleteTablets", vttablet.Alias) } clusterInfo.ClusterInstance.Keyspaces[0].Shards[0].Vttablets = nil return nil @@ -352,19 +351,16 @@ func ShardPrimaryTablet(t *testing.T, clusterInfo *VTOrcClusterInfo, keyspace *c if now.Sub(start) > time.Second*60 { assert.FailNow(t, "failed to elect primary before timeout") } - result, err := clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShard", fmt.Sprintf("%s/%s", keyspace.Name, shard.Name)) - assert.Nil(t, err) + si, err := clusterInfo.ClusterInstance.VtctldClientProcess.GetShard(keyspace.Name, shard.Name) + require.NoError(t, err) - var shardInfo topodatapb.Shard - err = json2.Unmarshal([]byte(result), &shardInfo) - assert.Nil(t, err) - if shardInfo.PrimaryAlias == nil { + if si.Shard.PrimaryAlias == nil { log.Warningf("Shard %v/%v has no primary yet, sleep for 1 second\n", keyspace.Name, shard.Name) time.Sleep(time.Second) continue } for _, tablet := range shard.Vttablets { - if tablet.Alias == topoproto.TabletAliasString(shardInfo.PrimaryAlias) { + if tablet.Alias == topoproto.TabletAliasString(si.Shard.PrimaryAlias) { return tablet } } @@ -381,12 +377,8 @@ func CheckPrimaryTablet(t *testing.T, clusterInfo *VTOrcClusterInfo, tablet *clu //log.Exitf("error") assert.FailNow(t, "failed to elect primary before timeout") } - result, err := clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", tablet.Alias) - require.NoError(t, err) - var tabletInfo topodatapb.Tablet - err = json2.Unmarshal([]byte(result), &tabletInfo) + tabletInfo, err := clusterInfo.ClusterInstance.VtctldClientProcess.GetTablet(tablet.Alias) require.NoError(t, err) - if topodatapb.TabletType_PRIMARY != tabletInfo.GetType() { log.Warningf("Tablet %v is not primary yet, sleep for 1 second\n", tablet.Alias) time.Sleep(time.Second) @@ -535,9 +527,9 @@ func validateTopology(t *testing.T, clusterInfo *VTOrcClusterInfo, pingTablets b var err error var output string if pingTablets { - output, err = clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("Validate", "--", "--ping-tablets=true") + output, err = clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("Validate", "--ping-tablets") } else { - output, err = clusterInfo.ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("Validate") + output, err = clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("Validate") } if err != nil { log.Warningf("Validate failed, retrying, output - %s", output) From d8ac5a8359f083c635021a5a7bae29ae2c940c5b Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Thu, 22 Feb 2024 11:07:22 +0530 Subject: [PATCH 05/16] Improve VTOrc startup flow (#15315) Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/api/api_test.go | 11 ++--- go/vt/vtorc/logic/tablet_discovery.go | 11 ++++- go/vt/vtorc/logic/tablet_discovery_test.go | 22 +++++++++ go/vt/vtorc/logic/vtorc.go | 56 ++++++++++------------ go/vt/vtorc/process/health.go | 4 +- 5 files changed, 64 insertions(+), 40 deletions(-) diff --git a/go/test/endtoend/vtorc/api/api_test.go b/go/test/endtoend/vtorc/api/api_test.go index 7b277fd7a0f..30e43dfc29a 100644 --- a/go/test/endtoend/vtorc/api/api_test.go +++ b/go/test/endtoend/vtorc/api/api_test.go @@ -37,21 +37,18 @@ func TestAPIEndpoints(t *testing.T) { utils.SetupVttabletsAndVTOrcs(t, clusterInfo, 2, 1, nil, cluster.VTOrcConfiguration{ PreventCrossDataCenterPrimaryFailover: true, RecoveryPeriodBlockSeconds: 5, - // The default topo refresh time is 3 seconds. We are intentionally making it slower for the test, so that we have time to verify - // the /debug/health output before and after the first refresh runs. - TopologyRefreshSeconds: 10, }, 1, "") keyspace := &clusterInfo.ClusterInstance.Keyspaces[0] shard0 := &keyspace.Shards[0] vtorc := clusterInfo.ClusterInstance.VTOrcProcesses[0] // Call API with retry to ensure VTOrc is up status, resp := utils.MakeAPICallRetry(t, vtorc, "/debug/health", func(code int, response string) bool { - return code == 0 + return code != 200 }) - // When VTOrc is up and hasn't run the topo-refresh, is should be healthy but HasDiscovered should be false. - assert.Equal(t, 500, status) + // Verify when VTOrc is healthy, it has also run the first discovery. + assert.Equal(t, 200, status) assert.Contains(t, resp, `"Healthy": true,`) - assert.Contains(t, resp, `"DiscoveredOnce": false`) + assert.Contains(t, resp, `"DiscoveredOnce": true`) // find primary from topo primary := utils.ShardPrimaryTablet(t, clusterInfo, keyspace, shard0) diff --git a/go/vt/vtorc/logic/tablet_discovery.go b/go/vt/vtorc/logic/tablet_discovery.go index dd2e65237bf..f08ab2c9c15 100644 --- a/go/vt/vtorc/logic/tablet_discovery.go +++ b/go/vt/vtorc/logic/tablet_discovery.go @@ -40,6 +40,7 @@ import ( "vitess.io/vitess/go/vt/vtorc/config" "vitess.io/vitess/go/vt/vtorc/db" "vitess.io/vitess/go/vt/vtorc/inst" + "vitess.io/vitess/go/vt/vtorc/process" "vitess.io/vitess/go/vt/vttablet/tmclient" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -64,16 +65,24 @@ func RegisterFlags(fs *pflag.FlagSet) { // OpenTabletDiscovery opens the vitess topo if enables and returns a ticker // channel for polling. func OpenTabletDiscovery() <-chan time.Time { - // TODO(sougou): If there's a shutdown signal, we have to close the topo. ts = topo.Open() tmc = tmclient.NewTabletManagerClient() // Clear existing cache and perform a new refresh. if _, err := db.ExecVTOrc("delete from vitess_tablet"); err != nil { log.Error(err) } + // We refresh all information from the topo once before we start the ticks to do it on a timer. + populateAllInformation() return time.Tick(time.Second * time.Duration(config.Config.TopoInformationRefreshSeconds)) //nolint SA1015: using time.Tick leaks the underlying ticker } +// populateAllInformation initializes all the information for VTOrc to function. +func populateAllInformation() { + refreshAllInformation() + // We have completed one full discovery cycle. We should update the process health. + process.FirstDiscoveryCycleComplete.Store(true) +} + // refreshAllTablets reloads the tablets from topo and discovers the ones which haven't been refreshed in a while func refreshAllTablets() { refreshTabletsUsing(func(tabletAlias string) { diff --git a/go/vt/vtorc/logic/tablet_discovery_test.go b/go/vt/vtorc/logic/tablet_discovery_test.go index 0e8ac72fabf..f79cecf9ff5 100644 --- a/go/vt/vtorc/logic/tablet_discovery_test.go +++ b/go/vt/vtorc/logic/tablet_discovery_test.go @@ -34,6 +34,7 @@ import ( "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vtorc/db" "vitess.io/vitess/go/vt/vtorc/inst" + "vitess.io/vitess/go/vt/vtorc/process" ) var ( @@ -342,3 +343,24 @@ func TestGetLockAction(t *testing.T) { }) } } + +// TestProcessHealth tests that the health of the process reflects that we have run the first discovery once correctly. +func TestProcessHealth(t *testing.T) { + require.False(t, process.FirstDiscoveryCycleComplete.Load()) + originalTs := ts + defer func() { + ts = originalTs + process.FirstDiscoveryCycleComplete.Store(false) + }() + // Verify in the beginning, we have the first DiscoveredOnce field false. + health, err := process.HealthTest() + require.NoError(t, err) + require.False(t, health.DiscoveredOnce) + ts = memorytopo.NewServer(context.Background(), cell1) + populateAllInformation() + require.True(t, process.FirstDiscoveryCycleComplete.Load()) + // Verify after we populate all information, we have the first DiscoveredOnce field true. + health, err = process.HealthTest() + require.NoError(t, err) + require.True(t, health.DiscoveredOnce) +} diff --git a/go/vt/vtorc/logic/vtorc.go b/go/vt/vtorc/logic/vtorc.go index f637956fbfd..66c5590831b 100644 --- a/go/vt/vtorc/logic/vtorc.go +++ b/go/vt/vtorc/logic/vtorc.go @@ -129,6 +129,7 @@ func closeVTOrc() { _ = inst.AuditOperation("shutdown", "", "Triggered via SIGTERM") // wait for the locks to be released waitForLocksRelease() + ts.Close() log.Infof("VTOrc closed") } @@ -335,8 +336,6 @@ func onHealthTick() { // nolint SA1015: using time.Tick leaks the underlying ticker func ContinuousDiscovery() { log.Infof("continuous discovery: setting up") - continuousDiscoveryStartTime := time.Now() - checkAndRecoverWaitPeriod := 3 * instancePollSecondsDuration() recentDiscoveryOperationKeys = cache.New(instancePollSecondsDuration(), time.Second) go handleDiscoveryRequests() @@ -351,10 +350,6 @@ func ContinuousDiscovery() { snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour) } - runCheckAndRecoverOperationsTimeRipe := func() bool { - return time.Since(continuousDiscoveryStartTime) >= checkAndRecoverWaitPeriod - } - go func() { _ = ometrics.InitMetrics() }() @@ -400,11 +395,7 @@ func ContinuousDiscovery() { } else { return } - if runCheckAndRecoverOperationsTimeRipe() { - CheckAndRecover() - } else { - log.Infof("Waiting for %+v seconds to pass before running failure detection/recovery", checkAndRecoverWaitPeriod.Seconds()) - } + CheckAndRecover() }() } }() @@ -415,27 +406,30 @@ func ContinuousDiscovery() { } }() case <-tabletTopoTick: - // Create a wait group - var wg sync.WaitGroup + refreshAllInformation() + } + } +} - // Refresh all keyspace information. - wg.Add(1) - go func() { - defer wg.Done() - RefreshAllKeyspacesAndShards() - }() +// refreshAllInformation refreshes both shard and tablet information. This is meant to be run on tablet topo ticks. +func refreshAllInformation() { + // Create a wait group + var wg sync.WaitGroup - // Refresh all tablets. - wg.Add(1) - go func() { - defer wg.Done() - refreshAllTablets() - }() + // Refresh all keyspace information. + wg.Add(1) + go func() { + defer wg.Done() + RefreshAllKeyspacesAndShards() + }() - // Wait for both the refreshes to complete - wg.Wait() - // We have completed one discovery cycle in the entirety of it. We should update the process health. - process.FirstDiscoveryCycleComplete.Store(true) - } - } + // Refresh all tablets. + wg.Add(1) + go func() { + defer wg.Done() + refreshAllTablets() + }() + + // Wait for both the refreshes to complete + wg.Wait() } diff --git a/go/vt/vtorc/process/health.go b/go/vt/vtorc/process/health.go index 22db89e1d56..a782b2edf14 100644 --- a/go/vt/vtorc/process/health.go +++ b/go/vt/vtorc/process/health.go @@ -108,7 +108,9 @@ func RegisterNode(nodeHealth *NodeHealth) (healthy bool, err error) { func HealthTest() (health *HealthStatus, err error) { cacheKey := util.ProcessToken.Hash if healthStatus, found := lastHealthCheckCache.Get(cacheKey); found { - return healthStatus.(*HealthStatus), nil + health = healthStatus.(*HealthStatus) + health.DiscoveredOnce = FirstDiscoveryCycleComplete.Load() + return } health = &HealthStatus{Healthy: false, Hostname: ThisHostname, Token: util.ProcessToken.Hash} From e5499807932015dc7fb31569f5557005cc3119db Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 22 Feb 2024 07:52:52 +0100 Subject: [PATCH 06/16] Skip for-loop alloc in `go/vt/discovery/healthcheck.go` (#15326) Signed-off-by: Tim Vaillancourt --- go/vt/discovery/healthcheck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 5d6a5e32662..5572e9a41d9 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -774,6 +774,7 @@ func FilterTargetsByKeyspaces(keyspaces []string, targets []*query.Target) []*qu func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*query.Target, requireServing bool) error { targets = FilterTargetsByKeyspaces(KeyspacesToWatch, targets) + var tabletHealths []*TabletHealth for { // We nil targets as we find them. allPresent := true @@ -782,7 +783,6 @@ func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*query. continue } - var tabletHealths []*TabletHealth if requireServing { tabletHealths = hc.GetHealthyTabletStats(target) } else { From c536bb7b26603931f6f2adac0065cf2a0c210869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Taylor?= Date: Thu, 22 Feb 2024 08:56:16 +0100 Subject: [PATCH 07/16] Column alias expanding on ORDER BY (#15302) Co-authored-by: Harshit Gangal Co-authored-by: Manan Gupta --- go/mysql/sqlerror/sql_error.go | 1 + .../queries/aggregation/aggregation_test.go | 36 +- .../vtgate/queries/dml/insert_test.go | 12 +- .../queries/lookup_queries/main_test.go | 2 +- .../endtoend/vtgate/queries/misc/misc_test.go | 4 +- .../vtgate/queries/orderby/orderby_test.go | 70 ++++ .../vtgate/queries/orderby/schema.sql | 9 + .../vtgate/queries/orderby/vschema.json | 8 + .../vtgate/queries/union/union_test.go | 9 +- go/vt/schemadiff/schema.go | 2 +- go/vt/vterrors/state.go | 1 + .../testdata/postprocess_cases.json | 134 +++++++ .../testdata/unsupported_cases.json | 5 - go/vt/vtgate/semantics/analyzer.go | 19 +- go/vt/vtgate/semantics/binder.go | 15 +- go/vt/vtgate/semantics/early_rewriter.go | 333 +++++++++++++++--- go/vt/vtgate/semantics/early_rewriter_test.go | 207 ++++++++--- go/vt/vtgate/semantics/errors.go | 26 +- 18 files changed, 749 insertions(+), 144 deletions(-) diff --git a/go/mysql/sqlerror/sql_error.go b/go/mysql/sqlerror/sql_error.go index 3ffbc7f9c8c..bebd9e41ca7 100644 --- a/go/mysql/sqlerror/sql_error.go +++ b/go/mysql/sqlerror/sql_error.go @@ -243,6 +243,7 @@ var stateToMysqlCode = map[vterrors.State]mysqlCode{ vterrors.WrongParametersToNativeFct: {num: ERWrongParametersToNativeFct, state: SSUnknownSQLState}, vterrors.KillDeniedError: {num: ERKillDenied, state: SSUnknownSQLState}, vterrors.BadNullError: {num: ERBadNullError, state: SSConstraintViolation}, + vterrors.InvalidGroupFuncUse: {num: ERInvalidGroupFuncUse, state: SSUnknownSQLState}, } func getStateToMySQLState(state vterrors.State) mysqlCode { diff --git a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go index 89b3d0c8c85..6f4dd01d4e2 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go @@ -73,7 +73,7 @@ func TestAggregateTypes(t *testing.T) { mcmp.AssertMatches("select val1 as a, count(*) from aggr_test group by a order by a", `[[VARCHAR("a") INT64(2)] [VARCHAR("b") INT64(1)] [VARCHAR("c") INT64(2)] [VARCHAR("d") INT64(1)] [VARCHAR("e") INT64(2)]]`) mcmp.AssertMatches("select val1 as a, count(*) from aggr_test group by a order by 2, a", `[[VARCHAR("b") INT64(1)] [VARCHAR("d") INT64(1)] [VARCHAR("a") INT64(2)] [VARCHAR("c") INT64(2)] [VARCHAR("e") INT64(2)]]`) mcmp.AssertMatches("select sum(val1) from aggr_test", `[[FLOAT64(0)]]`) - t.Run("Average for sharded keyspaces", func(t *testing.T) { + mcmp.Run("Average for sharded keyspaces", func(mcmp *utils.MySQLCompare) { utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") mcmp.AssertMatches("select avg(val1) from aggr_test", `[[FLOAT64(0)]]`) }) @@ -101,7 +101,7 @@ func TestEqualFilterOnScatter(t *testing.T) { workloads := []string{"oltp", "olap"} for _, workload := range workloads { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload)) mcmp.AssertMatches("select count(*) as a from aggr_test having 1 = 1", `[[INT64(5)]]`) @@ -177,7 +177,7 @@ func TestAggrOnJoin(t *testing.T) { mcmp.AssertMatches("select a.val1 from aggr_test a join t3 t on a.val2 = t.id7 group by a.val1 having count(*) = 4", `[[VARCHAR("a")]]`) - t.Run("Average in join for sharded", func(t *testing.T) { + mcmp.Run("Average in join for sharded", func(mcmp *utils.MySQLCompare) { utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") mcmp.AssertMatches(`select avg(a1.val2), avg(a2.val2) from aggr_test a1 join aggr_test a2 on a1.val2 = a2.id join t3 t on a2.val2 = t.id7`, "[[DECIMAL(1.5000) DECIMAL(1.0000)]]") @@ -196,7 +196,7 @@ func TestNotEqualFilterOnScatter(t *testing.T) { workloads := []string{"oltp", "olap"} for _, workload := range workloads { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload)) mcmp.AssertMatches("select count(*) as a from aggr_test having a != 5", `[]`) @@ -220,7 +220,7 @@ func TestLessFilterOnScatter(t *testing.T) { workloads := []string{"oltp", "olap"} for _, workload := range workloads { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload)) mcmp.AssertMatches("select count(*) as a from aggr_test having a < 10", `[[INT64(5)]]`) mcmp.AssertMatches("select count(*) as a from aggr_test having 1 < a", `[[INT64(5)]]`) @@ -243,7 +243,7 @@ func TestLessEqualFilterOnScatter(t *testing.T) { workloads := []string{"oltp", "olap"} for _, workload := range workloads { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload)) mcmp.AssertMatches("select count(*) as a from aggr_test having a <= 10", `[[INT64(5)]]`) @@ -267,7 +267,7 @@ func TestGreaterFilterOnScatter(t *testing.T) { workloads := []string{"oltp", "olap"} for _, workload := range workloads { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload)) mcmp.AssertMatches("select count(*) as a from aggr_test having a > 1", `[[INT64(5)]]`) @@ -291,7 +291,7 @@ func TestGreaterEqualFilterOnScatter(t *testing.T) { workloads := []string{"oltp", "olap"} for _, workload := range workloads { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload)) mcmp.AssertMatches("select count(*) as a from aggr_test having a >= 1", `[[INT64(5)]]`) @@ -326,7 +326,7 @@ func TestAggOnTopOfLimit(t *testing.T) { mcmp.Exec("insert into aggr_test(id, val1, val2) values(1,'a',6), (2,'a',1), (3,'b',1), (4,'c',3), (5,'c',4), (6,'b',null), (7,null,2), (8,null,null)") for _, workload := range []string{"oltp", "olap"} { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload)) mcmp.AssertMatches("select count(*) from (select id, val1 from aggr_test where val2 < 4 limit 2) as x", "[[INT64(2)]]") mcmp.AssertMatches("select count(val1) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2)]]") @@ -335,7 +335,7 @@ func TestAggOnTopOfLimit(t *testing.T) { mcmp.AssertMatches("select count(val2) from (select id, val2 from aggr_test where val2 is null limit 2) as x", "[[INT64(0)]]") mcmp.AssertMatches("select val1, count(*) from (select id, val1 from aggr_test where val2 < 4 order by val1 limit 2) as x group by val1", `[[NULL INT64(1)] [VARCHAR("a") INT64(1)]]`) mcmp.AssertMatchesNoOrder("select val1, count(val2) from (select val1, val2 from aggr_test limit 8) as x group by val1", `[[NULL INT64(1)] [VARCHAR("a") INT64(2)] [VARCHAR("b") INT64(1)] [VARCHAR("c") INT64(2)]]`) - t.Run("Average in sharded query", func(t *testing.T) { + mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") mcmp.AssertMatches("select avg(val2) from (select id, val2 from aggr_test where val2 is null limit 2) as x", "[[NULL]]") mcmp.AssertMatchesNoOrder("select val1, avg(val2) from (select val1, val2 from aggr_test limit 8) as x group by val1", `[[NULL DECIMAL(2.0000)] [VARCHAR("a") DECIMAL(3.5000)] [VARCHAR("b") DECIMAL(1.0000)] [VARCHAR("c") DECIMAL(3.5000)]]`) @@ -347,7 +347,7 @@ func TestAggOnTopOfLimit(t *testing.T) { mcmp.AssertMatches("select count(val1), sum(id) from (select id, val1 from aggr_test where val2 is null limit 2) as x", "[[INT64(1) DECIMAL(14)]]") mcmp.AssertMatches("select count(val2), sum(val2) from (select id, val2 from aggr_test where val2 is null limit 2) as x", "[[INT64(0) NULL]]") mcmp.AssertMatches("select val1, count(*), sum(id) from (select id, val1 from aggr_test where val2 < 4 order by val1 limit 2) as x group by val1", `[[NULL INT64(1) DECIMAL(7)] [VARCHAR("a") INT64(1) DECIMAL(2)]]`) - t.Run("Average in sharded query", func(t *testing.T) { + mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") mcmp.AssertMatches("select count(*), sum(val1), avg(val1) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2) FLOAT64(0) FLOAT64(0)]]") mcmp.AssertMatches("select count(val1), sum(id), avg(id) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2) DECIMAL(7) DECIMAL(3.5000)]]") @@ -363,13 +363,13 @@ func TestEmptyTableAggr(t *testing.T) { defer closer() for _, workload := range []string{"oltp", "olap"} { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", workload)) mcmp.AssertMatches(" select count(*) from t1 inner join t2 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]") mcmp.AssertMatches(" select count(*) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]") mcmp.AssertMatches(" select t1.`name`, count(*) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo' group by t1.`name`", "[]") mcmp.AssertMatches(" select t1.`name`, count(*) from t1 inner join t2 on (t1.t1_id = t2.id) where t1.value = 'foo' group by t1.`name`", "[]") - t.Run("Average in sharded query", func(t *testing.T) { + mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") mcmp.AssertMatches(" select count(t1.value) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]") mcmp.AssertMatches(" select avg(t1.value) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[NULL]]") @@ -380,12 +380,12 @@ func TestEmptyTableAggr(t *testing.T) { mcmp.Exec("insert into t1(t1_id, `name`, `value`, shardkey) values(1,'a1','foo',100), (2,'b1','foo',200), (3,'c1','foo',300), (4,'a1','foo',100), (5,'b1','bar',200)") for _, workload := range []string{"oltp", "olap"} { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", workload)) mcmp.AssertMatches(" select count(*) from t1 inner join t2 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]") mcmp.AssertMatches(" select count(*) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]") mcmp.AssertMatches(" select t1.`name`, count(*) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo' group by t1.`name`", "[]") - t.Run("Average in sharded query", func(t *testing.T) { + mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") mcmp.AssertMatches(" select count(t1.value) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]") mcmp.AssertMatches(" select avg(t1.value) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[NULL]]") @@ -434,7 +434,7 @@ func TestAggregateLeftJoin(t *testing.T) { mcmp.AssertMatches("SELECT sum(t2.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(1)]]`) mcmp.AssertMatches("SELECT count(*) FROM t2 LEFT JOIN t1 ON t1.t1_id = t2.id WHERE IFNULL(t1.name, 'NOTSET') = 'r'", `[[INT64(1)]]`) - t.Run("Average in sharded query", func(t *testing.T) { + mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") mcmp.AssertMatches("SELECT avg(t1.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(0.5000)]]`) mcmp.AssertMatches("SELECT avg(t2.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(1.0000)]]`) @@ -491,7 +491,7 @@ func TestScalarAggregate(t *testing.T) { mcmp.Exec("insert into aggr_test(id, val1, val2) values(1,'a',1), (2,'A',1), (3,'b',1), (4,'c',3), (5,'c',4)") mcmp.AssertMatches("select count(distinct val1) from aggr_test", `[[INT64(3)]]`) - t.Run("Average in sharded query", func(t *testing.T) { + mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") mcmp.AssertMatches("select avg(val1) from aggr_test", `[[FLOAT64(0)]]`) }) @@ -551,7 +551,7 @@ func TestComplexAggregation(t *testing.T) { mcmp.Exec(`SELECT shardkey + MIN(t1_id)+MAX(t1_id) FROM t1 GROUP BY shardkey`) mcmp.Exec(`SELECT name+COUNT(t1_id)+1 FROM t1 GROUP BY name`) mcmp.Exec(`SELECT COUNT(*)+shardkey+MIN(t1_id)+1+MAX(t1_id)*SUM(t1_id)+1+name FROM t1 GROUP BY shardkey, name`) - t.Run("Average in sharded query", func(t *testing.T) { + mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") mcmp.Exec(`SELECT COUNT(t1_id)+MAX(shardkey)+AVG(t1_id) FROM t1`) }) diff --git a/go/test/endtoend/vtgate/queries/dml/insert_test.go b/go/test/endtoend/vtgate/queries/dml/insert_test.go index 80d0602b898..ce052b7b2ba 100644 --- a/go/test/endtoend/vtgate/queries/dml/insert_test.go +++ b/go/test/endtoend/vtgate/queries/dml/insert_test.go @@ -38,7 +38,7 @@ func TestSimpleInsertSelect(t *testing.T) { mcmp.Exec("insert into u_tbl(id, num) values (1,2),(3,4)") for i, mode := range []string{"oltp", "olap"} { - t.Run(mode, func(t *testing.T) { + mcmp.Run(mode, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", mode)) qr := mcmp.Exec(fmt.Sprintf("insert into s_tbl(id, num) select id*%d, num*%d from s_tbl where id < 10", 10+i, 20+i)) @@ -65,7 +65,7 @@ func TestFailureInsertSelect(t *testing.T) { mcmp.Exec("insert into u_tbl(id, num) values (1,2),(3,4)") for _, mode := range []string{"oltp", "olap"} { - t.Run(mode, func(t *testing.T) { + mcmp.Run(mode, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", mode)) // primary key same @@ -127,7 +127,7 @@ func TestAutoIncInsertSelect(t *testing.T) { }} for _, tcase := range tcases { - t.Run(tcase.query, func(t *testing.T) { + mcmp.Run(tcase.query, func(mcmp *utils.MySQLCompare) { qr := utils.Exec(t, mcmp.VtConn, tcase.query) assert.EqualValues(t, tcase.expRowsAffected, qr.RowsAffected) assert.EqualValues(t, tcase.expInsertID, qr.InsertID) @@ -178,7 +178,7 @@ func TestAutoIncInsertSelectOlapMode(t *testing.T) { }} for _, tcase := range tcases { - t.Run(tcase.query, func(t *testing.T) { + mcmp.Run(tcase.query, func(mcmp *utils.MySQLCompare) { qr := utils.Exec(t, mcmp.VtConn, tcase.query) assert.EqualValues(t, tcase.expRowsAffected, qr.RowsAffected) assert.EqualValues(t, tcase.expInsertID, qr.InsertID) @@ -386,7 +386,7 @@ func TestInsertSelectUnshardedUsingSharded(t *testing.T) { mcmp.Exec("insert into s_tbl(id, num) values (1,2),(3,4)") for _, mode := range []string{"oltp", "olap"} { - t.Run(mode, func(t *testing.T) { + mcmp.Run(mode, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", mode)) qr := mcmp.Exec("insert into u_tbl(id, num) select id, num from s_tbl where s_tbl.id in (1,3)") assert.EqualValues(t, 2, qr.RowsAffected) @@ -453,7 +453,7 @@ func TestMixedCases(t *testing.T) { }} for _, tc := range tcases { - t.Run(tc.insQuery, func(t *testing.T) { + mcmp.Run(tc.insQuery, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, tc.insQuery) utils.AssertMatches(t, mcmp.VtConn, tc.selQuery, tc.exp) }) diff --git a/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go b/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go index 9486dc194ff..a587f124762 100644 --- a/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go +++ b/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go @@ -134,7 +134,7 @@ func TestLookupQueries(t *testing.T) { (3, 'monkey', 'monkey')`) for _, workload := range []string{"olap", "oltp"} { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, "set workload = "+workload) mcmp.AssertMatches("select id from user where lookup = 'apa'", "[[INT64(1)] [INT64(2)]]") diff --git a/go/test/endtoend/vtgate/queries/misc/misc_test.go b/go/test/endtoend/vtgate/queries/misc/misc_test.go index 0b7671c753d..472d725488c 100644 --- a/go/test/endtoend/vtgate/queries/misc/misc_test.go +++ b/go/test/endtoend/vtgate/queries/misc/misc_test.go @@ -311,7 +311,7 @@ func TestAnalyze(t *testing.T) { defer closer() for _, workload := range []string{"olap", "oltp"} { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", workload)) utils.Exec(t, mcmp.VtConn, "analyze table t1") utils.Exec(t, mcmp.VtConn, "analyze table uks.unsharded") @@ -344,7 +344,7 @@ func TestTransactionModeVar(t *testing.T) { }} for _, tcase := range tcases { - t.Run(tcase.setStmt, func(t *testing.T) { + mcmp.Run(tcase.setStmt, func(mcmp *utils.MySQLCompare) { if tcase.setStmt != "" { utils.Exec(t, mcmp.VtConn, tcase.setStmt) } diff --git a/go/test/endtoend/vtgate/queries/orderby/orderby_test.go b/go/test/endtoend/vtgate/queries/orderby/orderby_test.go index 43f800ee24c..993f7834301 100644 --- a/go/test/endtoend/vtgate/queries/orderby/orderby_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/orderby_test.go @@ -83,3 +83,73 @@ func TestOrderBy(t *testing.T) { mcmp.AssertMatches("select id1, id2 from t4 order by reverse(id2) desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(2) VARCHAR("Abc")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(1) VARCHAR("a")]]`) } } + +func TestOrderByComplex(t *testing.T) { + // tests written to try to trick the ORDER BY engine and planner + utils.SkipIfBinaryIsBelowVersion(t, 20, "vtgate") + + mcmp, closer := start(t) + defer closer() + + mcmp.Exec("insert into user(id, col, email) values(1,1,'a'), (2,2,'Abc'), (3,3,'b'), (4,4,'c'), (5,2,'test'), (6,1,'test'), (7,2,'a'), (8,3,'b'), (9,4,'c3'), (10,2,'d')") + + queries := []string{ + "select email, max(col) from user group by email order by col", + "select email, max(col) from user group by email order by col + 1", + "select email, max(col) from user group by email order by max(col)", + "select email, max(col) from user group by email order by max(col) + 1", + "select email, max(col) from user group by email order by min(col)", + "select email, max(col) as col from user group by email order by col", + "select email, max(col) as col from user group by email order by max(col)", + "select email, max(col) as col from user group by email order by col + 1", + "select email, max(col) as col from user group by email order by email + col", + "select email, max(col) as col from user group by email order by email + max(col)", + "select email, max(col) as col from user group by email order by email, col", + "select email, max(col) as xyz from user group by email order by email, xyz", + "select email, max(col) as xyz from user group by email order by email, max(xyz)", + "select email, max(col) as xyz from user group by email order by email, abs(xyz)", + "select email, max(col) as xyz from user group by email order by email, max(col)", + "select email, max(col) as xyz from user group by email order by email, abs(col)", + "select email, max(col) as xyz from user group by email order by xyz + email", + "select email, max(col) as xyz from user group by email order by abs(xyz) + email", + "select email, max(col) as xyz from user group by email order by abs(xyz)", + "select email, max(col) as xyz from user group by email order by abs(col)", + "select email, max(col) as max_col from user group by email order by max_col desc, length(email)", + "select email, max(col) as max_col, min(col) as min_col from user group by email order by max_col - min_col", + "select email, max(col) as col1, count(*) as col2 from user group by email order by col2 * col1", + "select email, sum(col) as sum_col from user group by email having sum_col > 10 order by sum_col / count(email)", + "select email, max(col) as max_col, char_length(email) as len_email from user group by email order by len_email, max_col desc", + "select email, max(col) as col_alias from user group by email order by case when col_alias > 100 then 0 else 1 end, col_alias", + "select email, count(*) as cnt, max(col) as max_col from user group by email order by cnt desc, max_col + cnt", + "select email, max(col) as max_col from user group by email order by if(max_col > 50, max_col, -max_col) desc", + "select email, max(col) as col, sum(col) as sum_col from user group by email order by col * sum_col desc", + "select email, max(col) as col, (select min(col) from user as u2 where u2.email = user.email) as min_col from user group by email order by col - min_col", + "select email, max(col) as max_col, (max(col) % 10) as mod_col from user group by email order by mod_col, max_col", + "select email, max(col) as 'value', count(email) as 'number' from user group by email order by 'number', 'value'", + "select email, max(col) as col, concat('email: ', email, ' col: ', max(col)) as complex_alias from user group by email order by complex_alias desc", + "select email, max(col) as max_col from user group by email union select email, min(col) as min_col from user group by email order by email", + "select email, max(col) as col from user where col > 50 group by email order by col desc", + "select email, max(col) as col from user group by email order by length(email), col", + "select email, max(col) as max_col, substring(email, 1, 3) as sub_email from user group by email order by sub_email, max_col desc", + "select email, max(col) as max_col from user group by email order by reverse(email), max_col", + "select email, max(col) as max_col from user group by email having max_col > avg(max_col) order by max_col desc", + "select email, count(*) as count, max(col) as max_col from user group by email order by count * max_col desc", + "select email, max(col) as col_alias from user group by email order by col_alias limit 10", + "select email, max(col) as col from user group by email order by col desc, email", + "select concat(email, ' ', max(col)) as combined from user group by email order by combined desc", + "select email, max(col) as max_col from user group by email order by ascii(email), max_col", + "select email, char_length(email) as email_length, max(col) as max_col from user group by email order by email_length desc, max_col", + "select email, max(col) as col from user group by email having col between 10 and 100 order by col", + "select email, max(col) as max_col, min(col) as min_col from user group by email order by max_col + min_col desc", + "select email, max(col) as 'max', count(*) as 'count' from user group by email order by 'max' desc, 'count'", + "select email, max(col) as max_col from (select email, col from user where col > 20) as filtered group by email order by max_col", + "select a.email, a.max_col from (select email, max(col) as max_col from user group by email) as a order by a.max_col desc", + "select email, max(col) as max_col from user where email like 'a%' group by email order by max_col, email", + } + + for _, query := range queries { + mcmp.Run(query, func(mcmp *utils.MySQLCompare) { + _, _ = mcmp.ExecAllowAndCompareError(query) + }) + } +} diff --git a/go/test/endtoend/vtgate/queries/orderby/schema.sql b/go/test/endtoend/vtgate/queries/orderby/schema.sql index 8f0131db357..efaedc14754 100644 --- a/go/test/endtoend/vtgate/queries/orderby/schema.sql +++ b/go/test/endtoend/vtgate/queries/orderby/schema.sql @@ -27,3 +27,12 @@ create table t4_id2_idx ) Engine = InnoDB DEFAULT charset = utf8mb4 COLLATE = utf8mb4_general_ci; + +create table user +( + id bigint primary key, + col bigint, + email varchar(20) +) Engine = InnoDB + DEFAULT charset = utf8mb4 + COLLATE = utf8mb4_general_ci; \ No newline at end of file diff --git a/go/test/endtoend/vtgate/queries/orderby/vschema.json b/go/test/endtoend/vtgate/queries/orderby/vschema.json index 14418850a35..771676de4b9 100644 --- a/go/test/endtoend/vtgate/queries/orderby/vschema.json +++ b/go/test/endtoend/vtgate/queries/orderby/vschema.json @@ -66,6 +66,14 @@ "name": "unicode_loose_md5" } ] + }, + "user": { + "column_vindexes": [ + { + "column": "id", + "name": "hash" + } + ] } } } \ No newline at end of file diff --git a/go/test/endtoend/vtgate/queries/union/union_test.go b/go/test/endtoend/vtgate/queries/union/union_test.go index 898f8b1d659..d91ea3c4073 100644 --- a/go/test/endtoend/vtgate/queries/union/union_test.go +++ b/go/test/endtoend/vtgate/queries/union/union_test.go @@ -57,7 +57,7 @@ func TestUnionDistinct(t *testing.T) { mcmp.Exec("insert into t2(id3, id4) values (2, 3), (3, 4), (4,4), (5,5)") for _, workload := range []string{"oltp", "olap"} { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, "set workload = "+workload) mcmp.AssertMatches("select 1 union select null", "[[INT64(1)] [NULL]]") mcmp.AssertMatches("select null union select null", "[[NULL]]") @@ -69,10 +69,7 @@ func TestUnionDistinct(t *testing.T) { mcmp.AssertMatchesNoOrder("select id1 from t1 where id1 = 1 union select 452 union select id1 from t1 where id1 = 4", "[[INT64(1)] [INT64(452)] [INT64(4)]]") mcmp.AssertMatchesNoOrder("select id1, id2 from t1 union select 827, 452 union select id3,id4 from t2", "[[INT64(4) INT64(4)] [INT64(1) INT64(1)] [INT64(2) INT64(2)] [INT64(3) INT64(3)] [INT64(827) INT64(452)] [INT64(2) INT64(3)] [INT64(3) INT64(4)] [INT64(5) INT64(5)]]") - t.Run("skipped for now", func(t *testing.T) { - t.Skip() - mcmp.AssertMatches("select 1 from dual where 1 IN (select 1 as col union select 2)", "[[INT64(1)]]") - }) + mcmp.AssertMatches("select 1 from dual where 1 IN (select 1 as col union select 2)", "[[INT64(1)]]") if utils.BinaryIsAtLeastAtVersion(19, "vtgate") { mcmp.AssertMatches(`SELECT 1 from t1 UNION SELECT 2 from t1`, `[[INT64(1)] [INT64(2)]]`) mcmp.AssertMatches(`SELECT 5 from t1 UNION SELECT 6 from t1`, `[[INT64(5)] [INT64(6)]]`) @@ -97,7 +94,7 @@ func TestUnionAll(t *testing.T) { mcmp.Exec("insert into t2(id3, id4) values(3, 3), (4, 4)") for _, workload := range []string{"oltp", "olap"} { - t.Run(workload, func(t *testing.T) { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { utils.Exec(t, mcmp.VtConn, "set workload = "+workload) // union all between two selectuniqueequal mcmp.AssertMatches("select id1 from t1 where id1 = 1 union all select id1 from t1 where id1 = 4", "[[INT64(1)]]") diff --git a/go/vt/schemadiff/schema.go b/go/vt/schemadiff/schema.go index 084b703b14f..e3782fdbf0b 100644 --- a/go/vt/schemadiff/schema.go +++ b/go/vt/schemadiff/schema.go @@ -1058,7 +1058,7 @@ func (s *Schema) ValidateViewReferences() error { Column: e.Column, Ambiguous: true, } - case *semantics.ColumnNotFoundError: + case semantics.ColumnNotFoundError: return &InvalidColumnReferencedInViewError{ View: view.Name(), Column: e.Column.Name.String(), diff --git a/go/vt/vterrors/state.go b/go/vt/vterrors/state.go index 00d64dd39a5..2b0ada0bc6d 100644 --- a/go/vt/vterrors/state.go +++ b/go/vt/vterrors/state.go @@ -48,6 +48,7 @@ const ( WrongValue WrongArguments BadNullError + InvalidGroupFuncUse // failed precondition NoDB diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json index 9093fe2f390..0b0c0658175 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json @@ -2235,5 +2235,139 @@ "user.user" ] } + }, + { + "comment": "ORDER BY literal works fine even when the columns have the same name", + "query": "select a.id, b.id from user as a, user_extra as b union all select 1, 2 order by 1", + "plan": { + "QueryType": "SELECT", + "Original": "select a.id, b.id from user as a, user_extra as b union all select 1, 2 order by 1", + "Instructions": { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(0|2) ASC", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Concatenate", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,L:1", + "TableName": "`user`_user_extra", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select a.id, weight_string(a.id) from `user` as a where 1 != 1", + "Query": "select a.id, weight_string(a.id) from `user` as a", + "Table": "`user`" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select b.id from user_extra as b where 1 != 1", + "Query": "select b.id from user_extra as b", + "Table": "user_extra" + } + ] + }, + { + "OperatorType": "Route", + "Variant": "Reference", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select 1, 2, weight_string(1) from dual where 1 != 1", + "Query": "select 1, 2, weight_string(1) from dual", + "Table": "dual" + } + ] + } + ] + }, + "TablesUsed": [ + "main.dual", + "user.user", + "user.user_extra" + ] + } + }, + { + "comment": "ORDER BY literal works fine even when the columns have the same name", + "query": "select a.id, b.id from user as a, user_extra as b union all select 1, 2 order by 2", + "plan": { + "QueryType": "SELECT", + "Original": "select a.id, b.id from user as a, user_extra as b union all select 1, 2 order by 2", + "Instructions": { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(1|2) ASC", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Concatenate", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,R:1", + "TableName": "`user`_user_extra", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select a.id from `user` as a where 1 != 1", + "Query": "select a.id from `user` as a", + "Table": "`user`" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select b.id, weight_string(b.id) from user_extra as b where 1 != 1", + "Query": "select b.id, weight_string(b.id) from user_extra as b", + "Table": "user_extra" + } + ] + }, + { + "OperatorType": "Route", + "Variant": "Reference", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select 1, 2, weight_string(2) from dual where 1 != 1", + "Query": "select 1, 2, weight_string(2) from dual", + "Table": "dual" + } + ] + } + ] + }, + "TablesUsed": [ + "main.dual", + "user.user", + "user.user_extra" + ] + } } ] diff --git a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json index 5c8bcad0c57..8007a0e1a0a 100644 --- a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json @@ -179,11 +179,6 @@ "query": "select id2 from user uu where id in (select id from user where id = uu.id and user.col in (select col from (select col, id, user_id from user_extra where user_id = 5) uu where uu.user_id = uu.id))", "plan": "VT12001: unsupported: correlated subquery is only supported for EXISTS" }, - { - "comment": "rewrite of 'order by 2' that becomes 'order by id', leading to ambiguous binding.", - "query": "select a.id, b.id from user as a, user_extra as b union select 1, 2 order by 2", - "plan": "Column 'id' in field list is ambiguous" - }, { "comment": "unsupported with clause in delete statement", "query": "with x as (select * from user) delete from x", diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index cf2b3300208..0fe9f4a934e 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -69,10 +69,12 @@ func (a *analyzer) lateInit() { a.binder = newBinder(a.scoper, a, a.tables, a.typer) a.scoper.binder = a.binder a.rewriter = &earlyRewriter{ - env: a.si.Environment(), - scoper: a.scoper, binder: a.binder, + scoper: a.scoper, expandedColumns: map[sqlparser.TableName][]*sqlparser.ColName{}, + env: a.si.Environment(), + aliasMapCache: map[*sqlparser.Select]map[string]exprContainer{}, + reAnalyze: a.lateAnalyze, } } @@ -232,10 +234,6 @@ func (a *analyzer) analyzeUp(cursor *sqlparser.Cursor) bool { return false } - if err := a.scoper.up(cursor); err != nil { - a.setError(err) - return false - } if err := a.tables.up(cursor); err != nil { a.setError(err) return false @@ -256,6 +254,11 @@ func (a *analyzer) analyzeUp(cursor *sqlparser.Cursor) bool { return true } + if err := a.scoper.up(cursor); err != nil { + a.setError(err) + return false + } + a.leaveProjection(cursor) return a.shouldContinue() } @@ -348,6 +351,10 @@ func (a *analyzer) analyze(statement sqlparser.Statement) error { a.lateInit() + return a.lateAnalyze(statement) +} + +func (a *analyzer) lateAnalyze(statement sqlparser.SQLNode) error { _ = sqlparser.Rewrite(statement, a.analyzeDown, a.analyzeUp) return a.err } diff --git a/go/vt/vtgate/semantics/binder.go b/go/vt/vtgate/semantics/binder.go index f5e7d3c6297..b010649e067 100644 --- a/go/vt/vtgate/semantics/binder.go +++ b/go/vt/vtgate/semantics/binder.go @@ -57,7 +57,8 @@ func newBinder(scoper *scoper, org originable, tc *tableCollector, typer *typer) } func (b *binder) up(cursor *sqlparser.Cursor) error { - switch node := cursor.Node().(type) { + node := cursor.Node() + switch node := node.(type) { case *sqlparser.Subquery: currScope := b.scoper.currentScope() b.setSubQueryDependencies(node, currScope) @@ -65,7 +66,7 @@ func (b *binder) up(cursor *sqlparser.Cursor) error { currScope := b.scoper.currentScope() for _, ident := range node.Using { name := sqlparser.NewColName(ident.String()) - deps, err := b.resolveColumn(name, currScope, true) + deps, err := b.resolveColumn(name, currScope, true, true) if err != nil { return err } @@ -73,7 +74,7 @@ func (b *binder) up(cursor *sqlparser.Cursor) error { } case *sqlparser.ColName: currentScope := b.scoper.currentScope() - deps, err := b.resolveColumn(node, currentScope, false) + deps, err := b.resolveColumn(node, currentScope, false, true) if err != nil { if deps.direct.IsEmpty() || !strings.HasSuffix(err.Error(), "is ambiguous") || @@ -185,7 +186,7 @@ func (b *binder) rewriteJoinUsingColName(deps dependency, node *sqlparser.ColNam return dependency{}, err } node.Qualifier = name - deps, err = b.resolveColumn(node, currentScope, false) + deps, err = b.resolveColumn(node, currentScope, false, true) if err != nil { return dependency{}, err } @@ -226,7 +227,7 @@ func (b *binder) setSubQueryDependencies(subq *sqlparser.Subquery, currScope *sc b.direct[subq] = subqDirectDeps.KeepOnly(tablesToKeep) } -func (b *binder) resolveColumn(colName *sqlparser.ColName, current *scope, allowMulti bool) (dependency, error) { +func (b *binder) resolveColumn(colName *sqlparser.ColName, current *scope, allowMulti, singleTableFallBack bool) (dependency, error) { var thisDeps dependencies first := true var tableName *sqlparser.TableName @@ -248,7 +249,7 @@ func (b *binder) resolveColumn(colName *sqlparser.ColName, current *scope, allow } else if err != nil { return dependency{}, err } - if current.parent == nil && len(current.tables) == 1 && first && colName.Qualifier.IsEmpty() { + if current.parent == nil && len(current.tables) == 1 && first && colName.Qualifier.IsEmpty() && singleTableFallBack { // if this is the top scope, and we still haven't been able to find a match, we know we are about to fail // we can check this last scope and see if there is a single table. if there is just one table in the scope // we assume that the column is meant to come from this table. @@ -263,7 +264,7 @@ func (b *binder) resolveColumn(colName *sqlparser.ColName, current *scope, allow first = false current = current.parent } - return dependency{}, ShardedError{&ColumnNotFoundError{Column: colName, Table: tableName}} + return dependency{}, ShardedError{ColumnNotFoundError{Column: colName, Table: tableName}} } func (b *binder) resolveColumnInScope(current *scope, expr *sqlparser.ColName, allowMulti bool) (dependencies, error) { diff --git a/go/vt/vtgate/semantics/early_rewriter.go b/go/vt/vtgate/semantics/early_rewriter.go index ed1f30c670e..646b5b71e41 100644 --- a/go/vt/vtgate/semantics/early_rewriter.go +++ b/go/vt/vtgate/semantics/early_rewriter.go @@ -34,6 +34,12 @@ type earlyRewriter struct { warning string expandedColumns map[sqlparser.TableName][]*sqlparser.ColName env *vtenv.Environment + aliasMapCache map[*sqlparser.Select]map[string]exprContainer + + // reAnalyze is used when we are running in the late stage, after the other parts of semantic analysis + // have happened, and we are introducing or changing the AST. We invoke it so all parts of the query have been + // typed, scoped and bound correctly + reAnalyze func(n sqlparser.SQLNode) error } func (r *earlyRewriter) down(cursor *sqlparser.Cursor) error { @@ -44,14 +50,6 @@ func (r *earlyRewriter) down(cursor *sqlparser.Cursor) error { return r.handleSelectExprs(cursor, node) case *sqlparser.JoinTableExpr: r.handleJoinTableExprDown(node) - case sqlparser.OrderBy: - r.clause = "order clause" - iter := &orderByIterator{ - node: node, - idx: -1, - } - - return r.handleOrderByAndGroupBy(cursor.Parent(), iter) case *sqlparser.OrExpr: rewriteOrExpr(r.env, cursor, node) case *sqlparser.AndExpr: @@ -64,7 +62,7 @@ func (r *earlyRewriter) down(cursor *sqlparser.Cursor) error { node: node, idx: -1, } - return r.handleOrderByAndGroupBy(cursor.Parent(), iter) + return r.handleGroupBy(cursor.Parent(), iter) case *sqlparser.ComparisonExpr: return handleComparisonExpr(cursor, node) case *sqlparser.With: @@ -77,6 +75,26 @@ func (r *earlyRewriter) down(cursor *sqlparser.Cursor) error { return nil } +func (r *earlyRewriter) up(cursor *sqlparser.Cursor) error { + switch node := cursor.Node().(type) { + case *sqlparser.JoinTableExpr: + return r.handleJoinTableExprUp(node) + case *sqlparser.AliasedTableExpr: + // this rewriting is done in the `up` phase, because we need the vindex hints to have been + // processed while collecting the tables. + return removeVindexHints(node) + case sqlparser.OrderBy: + r.clause = "order clause" + iter := &orderByIterator{ + node: node, + idx: -1, + r: r, + } + return r.handleOrderBy(cursor.Parent(), iter) + } + return nil +} + func handleDelete(del *sqlparser.Delete) error { // When we do not have any target, it is a single table delete. // In a single table delete, the table references is always a single aliased table expression. @@ -144,18 +162,6 @@ func rewriteNotExpr(cursor *sqlparser.Cursor, node *sqlparser.NotExpr) { cursor.Replace(cmp) } -func (r *earlyRewriter) up(cursor *sqlparser.Cursor) error { - switch node := cursor.Node().(type) { - case *sqlparser.JoinTableExpr: - return r.handleJoinTableExprUp(node) - case *sqlparser.AliasedTableExpr: - // this rewriting is done in the `up` phase, because we need the vindex hints to have been - // processed while collecting the tables. - return removeVindexHints(node) - } - return nil -} - func (r *earlyRewriter) handleJoinTableExprUp(join *sqlparser.JoinTableExpr) error { // this rewriting is done in the `up` phase, because we need the scope to have been // filled in with the available tables @@ -170,16 +176,8 @@ func (r *earlyRewriter) handleJoinTableExprUp(join *sqlparser.JoinTableExpr) err // since the binder has already been over the join, we need to invoke it again, so it // can bind columns to the right tables - sqlparser.Rewrite(join.Condition.On, nil, func(cursor *sqlparser.Cursor) bool { - innerErr := r.binder.up(cursor) - if innerErr == nil { - return true - } - err = innerErr - return false - }) - return err + return r.reAnalyze(join.Condition.On) } // removeVindexHints removes the vindex hints from the aliased table expression provided. @@ -207,7 +205,7 @@ func (r *earlyRewriter) handleWhereClause(node *sqlparser.Where, parent sqlparse if node.Type != sqlparser.HavingClause { return nil } - expr, err := r.rewriteAliasesInOrderByHavingAndGroupBy(node.Expr, sel) + expr, err := r.rewriteAliasesInHavingAndGroupBy(node.Expr, sel) if err != nil { return err } @@ -237,6 +235,7 @@ func (r *earlyRewriter) handleJoinTableExprDown(node *sqlparser.JoinTableExpr) { type orderByIterator struct { node sqlparser.OrderBy idx int + r *earlyRewriter } func (it *orderByIterator) next() sqlparser.Expr { @@ -249,7 +248,7 @@ func (it *orderByIterator) next() sqlparser.Expr { return it.node[it.idx].Expr } -func (it *orderByIterator) replace(e sqlparser.Expr) error { +func (it *orderByIterator) replace(e sqlparser.Expr) (err error) { if it.idx >= len(it.node) { return vterrors.VT13001("went past the last item") } @@ -285,13 +284,57 @@ type iterator interface { replace(e sqlparser.Expr) error } -func (r *earlyRewriter) replaceLiteralsInOrderByGroupBy(e sqlparser.Expr, iter iterator) (bool, error) { +func (r *earlyRewriter) replaceLiteralsInOrderBy(e sqlparser.Expr, iter iterator) (bool, error) { + lit := getIntLiteral(e) + if lit == nil { + return false, nil + } + + newExpr, recheck, err := r.rewriteOrderByExpr(lit) + if err != nil { + return false, err + } + + if getIntLiteral(newExpr) == nil { + coll, ok := e.(*sqlparser.CollateExpr) + if ok { + coll.Expr = newExpr + newExpr = coll + } + } else { + // the expression is still a literal int. that means that we don't really need to sort by it. + // we'll just replace the number with a string instead, just like mysql would do in this situation + // mysql> explain select 1 as foo from user group by 1; + // + // mysql> show warnings; + // +-------+------+-----------------------------------------------------------------+ + // | Level | Code | Message | + // +-------+------+-----------------------------------------------------------------+ + // | Note | 1003 | /* select#1 */ select 1 AS `foo` from `test`.`user` group by '' | + // +-------+------+-----------------------------------------------------------------+ + newExpr = sqlparser.NewStrLiteral("") + } + + err = iter.replace(newExpr) + if err != nil { + return false, err + } + if recheck { + err = r.reAnalyze(newExpr) + } + if err != nil { + return false, err + } + return true, nil +} + +func (r *earlyRewriter) replaceLiteralsInGroupBy(e sqlparser.Expr, iter iterator) (bool, error) { lit := getIntLiteral(e) if lit == nil { return false, nil } - newExpr, err := r.rewriteOrderByExpr(lit) + newExpr, err := r.rewriteGroupByExpr(lit) if err != nil { return false, err } @@ -341,7 +384,41 @@ func getIntLiteral(e sqlparser.Expr) *sqlparser.Literal { } // handleOrderBy processes the ORDER BY clause. -func (r *earlyRewriter) handleOrderByAndGroupBy(parent sqlparser.SQLNode, iter iterator) error { +func (r *earlyRewriter) handleOrderBy(parent sqlparser.SQLNode, iter iterator) error { + stmt, ok := parent.(sqlparser.SelectStatement) + if !ok { + return nil + } + + sel := sqlparser.GetFirstSelect(stmt) + for e := iter.next(); e != nil; e = iter.next() { + lit, err := r.replaceLiteralsInOrderBy(e, iter) + if err != nil { + return err + } + if lit { + continue + } + + expr, err := r.rewriteAliasesInOrderBy(e, sel) + if err != nil { + return err + } + + if err = iter.replace(expr); err != nil { + return err + } + + if err = r.reAnalyze(expr); err != nil { + return err + } + } + + return nil +} + +// handleGroupBy processes the GROUP BY clause. +func (r *earlyRewriter) handleGroupBy(parent sqlparser.SQLNode, iter iterator) error { stmt, ok := parent.(sqlparser.SelectStatement) if !ok { return nil @@ -349,14 +426,14 @@ func (r *earlyRewriter) handleOrderByAndGroupBy(parent sqlparser.SQLNode, iter i sel := sqlparser.GetFirstSelect(stmt) for e := iter.next(); e != nil; e = iter.next() { - lit, err := r.replaceLiteralsInOrderByGroupBy(e, iter) + lit, err := r.replaceLiteralsInGroupBy(e, iter) if err != nil { return err } if lit { continue } - expr, err := r.rewriteAliasesInOrderByHavingAndGroupBy(e, sel) + expr, err := r.rewriteAliasesInHavingAndGroupBy(e, sel) if err != nil { return err } @@ -375,7 +452,7 @@ func (r *earlyRewriter) handleOrderByAndGroupBy(parent sqlparser.SQLNode, iter i // in SELECT points to that expression, not any table column. // - However, if the aliased expression is an aggregation and the column identifier in // the HAVING/ORDER BY clause is inside an aggregation function, the rule does not apply. -func (r *earlyRewriter) rewriteAliasesInOrderByHavingAndGroupBy(node sqlparser.Expr, sel *sqlparser.Select) (expr sqlparser.Expr, err error) { +func (r *earlyRewriter) rewriteAliasesInHavingAndGroupBy(node sqlparser.Expr, sel *sqlparser.Select) (expr sqlparser.Expr, err error) { type ExprContainer struct { expr sqlparser.Expr ambiguous bool @@ -467,7 +544,176 @@ func (r *earlyRewriter) rewriteAliasesInOrderByHavingAndGroupBy(node sqlparser.E return } -func (r *earlyRewriter) rewriteOrderByExpr(node *sqlparser.Literal) (sqlparser.Expr, error) { +// rewriteAliasesInOrderBy rewrites columns in the ORDER BY and HAVING clauses to use aliases +// from the SELECT expressions when applicable, following MySQL scoping rules: +// - A column identifier without a table qualifier that matches an alias introduced +// in SELECT points to that expression, not any table column. +// - However, if the aliased expression is an aggregation and the column identifier in +// the HAVING/ORDER BY clause is inside an aggregation function, the rule does not apply. +func (r *earlyRewriter) rewriteAliasesInOrderBy(node sqlparser.Expr, sel *sqlparser.Select) (expr sqlparser.Expr, err error) { + currentScope := r.scoper.currentScope() + if currentScope.isUnion { + // It is not safe to rewrite order by clauses in unions. + return node, nil + } + + aliases := r.getAliasMap(sel) + insideAggr := false + dontEnterSubquery := func(node, _ sqlparser.SQLNode) bool { + switch node.(type) { + case *sqlparser.Subquery: + return false + case sqlparser.AggrFunc: + insideAggr = true + } + + _, isSubq := node.(*sqlparser.Subquery) + return !isSubq + } + output := sqlparser.CopyOnRewrite(node, dontEnterSubquery, func(cursor *sqlparser.CopyOnWriteCursor) { + var col *sqlparser.ColName + + switch node := cursor.Node().(type) { + case sqlparser.AggrFunc: + insideAggr = false + return + case *sqlparser.ColName: + col = node + default: + return + } + + if !col.Qualifier.IsEmpty() { + // we are only interested in columns not qualified by table names + return + } + + item, found := aliases[col.Name.Lowered()] + if !found { + // if there is no matching alias, there is no rewriting needed + return + } + + topLevel := col == node + if !topLevel && r.isColumnOnTable(col, currentScope) { + // we only want to replace columns that are not coming from the table + return + } + + if item.ambiguous { + err = &AmbiguousColumnError{Column: sqlparser.String(col)} + } else if insideAggr && sqlparser.ContainsAggregation(item.expr) { + err = &InvalidUserOfGroupFunction{} + } + if err != nil { + cursor.StopTreeWalk() + return + } + + cursor.Replace(sqlparser.CloneExpr(item.expr)) + }, nil) + + expr = output.(sqlparser.Expr) + return +} + +func (r *earlyRewriter) isColumnOnTable(col *sqlparser.ColName, currentScope *scope) bool { + if !currentScope.stmtScope && currentScope.parent != nil { + currentScope = currentScope.parent + } + _, err := r.binder.resolveColumn(col, currentScope, false, false) + return err == nil +} + +func (r *earlyRewriter) getAliasMap(sel *sqlparser.Select) (aliases map[string]exprContainer) { + var found bool + aliases, found = r.aliasMapCache[sel] + if found { + return + } + aliases = map[string]exprContainer{} + for _, e := range sel.SelectExprs { + ae, ok := e.(*sqlparser.AliasedExpr) + if !ok { + continue + } + + var alias string + + item := exprContainer{expr: ae.Expr} + if ae.As.NotEmpty() { + alias = ae.As.Lowered() + } else if col, ok := ae.Expr.(*sqlparser.ColName); ok { + alias = col.Name.Lowered() + } + + if old, alreadyExists := aliases[alias]; alreadyExists && !sqlparser.Equals.Expr(old.expr, item.expr) { + item.ambiguous = true + } + + aliases[alias] = item + } + return aliases +} + +type exprContainer struct { + expr sqlparser.Expr + ambiguous bool +} + +func (r *earlyRewriter) rewriteOrderByExpr(node *sqlparser.Literal) (expr sqlparser.Expr, needReAnalysis bool, err error) { + scope, found := r.scoper.specialExprScopes[node] + if !found { + return node, false, nil + } + num, err := strconv.Atoi(node.Val) + if err != nil { + return nil, false, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "error parsing column number: %s", node.Val) + } + + stmt, isSel := scope.stmt.(*sqlparser.Select) + if !isSel { + return nil, false, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "error invalid statement type, expect Select, got: %T", scope.stmt) + } + + if num < 1 || num > len(stmt.SelectExprs) { + return nil, false, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.BadFieldError, "Unknown column '%d' in '%s'", num, r.clause) + } + + // We loop like this instead of directly accessing the offset, to make sure there are no unexpanded `*` before + for i := 0; i < num; i++ { + if _, ok := stmt.SelectExprs[i].(*sqlparser.AliasedExpr); !ok { + return nil, false, vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "cannot use column offsets in %s when using `%s`", r.clause, sqlparser.String(stmt.SelectExprs[i])) + } + } + + colOffset := num - 1 + aliasedExpr, ok := stmt.SelectExprs[colOffset].(*sqlparser.AliasedExpr) + if !ok { + return nil, false, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "don't know how to handle %s", sqlparser.String(node)) + } + + if scope.isUnion { + colName := sqlparser.NewColName(aliasedExpr.ColumnName()) + vtabl, ok := scope.tables[0].(*vTableInfo) + if !ok { + panic("BUG: not expected") + } + + // since column names can be ambiguous here, we want to do the binding by offset and not by column name + allColExprs := vtabl.cols[colOffset] + direct, recursive, typ := r.binder.org.depsForExpr(allColExprs) + r.binder.direct[colName] = direct + r.binder.recursive[colName] = recursive + r.binder.typer.m[colName] = typ + + return colName, false, nil + } + + return realCloneOfColNames(aliasedExpr.Expr, false), true, nil +} + +func (r *earlyRewriter) rewriteGroupByExpr(node *sqlparser.Literal) (sqlparser.Expr, error) { scope, found := r.scoper.specialExprScopes[node] if !found { return node, nil @@ -499,13 +745,8 @@ func (r *earlyRewriter) rewriteOrderByExpr(node *sqlparser.Literal) (sqlparser.E } if scope.isUnion { - col, isCol := aliasedExpr.Expr.(*sqlparser.ColName) - - if aliasedExpr.As.IsEmpty() && isCol { - return sqlparser.NewColName(col.Name.String()), nil - } - - return sqlparser.NewColName(aliasedExpr.ColumnName()), nil + colName := sqlparser.NewColName(aliasedExpr.ColumnName()) + return colName, nil } return realCloneOfColNames(aliasedExpr.Expr, false), nil diff --git a/go/vt/vtgate/semantics/early_rewriter_test.go b/go/vt/vtgate/semantics/early_rewriter_test.go index e681f722b1d..cf93a52447c 100644 --- a/go/vt/vtgate/semantics/early_rewriter_test.go +++ b/go/vt/vtgate/semantics/early_rewriter_test.go @@ -304,42 +304,84 @@ func TestRewriteJoinUsingColumns(t *testing.T) { } -func TestOrderByGroupByLiteral(t *testing.T) { +func TestGroupByLiteral(t *testing.T) { schemaInfo := &FakeSI{ Tables: map[string]*vindexes.Table{}, } cDB := "db" tcases := []struct { - sql string - expSQL string - expErr string + sql string + expSQL string + expDeps TableSet + expErr string }{{ - sql: "select 1 as id from t1 order by 1", - expSQL: "select 1 as id from t1 order by '' asc", + sql: "select t1.col from t1 group by 1", + expSQL: "select t1.col from t1 group by t1.col", + expDeps: TS0, }, { - sql: "select t1.col from t1 order by 1", - expSQL: "select t1.col from t1 order by t1.col asc", + sql: "select t1.col as xyz from t1 group by 1", + expSQL: "select t1.col as xyz from t1 group by t1.col", + expDeps: TS0, }, { - sql: "select t1.col from t1 order by 1.0", - expSQL: "select t1.col from t1 order by 1.0 asc", + sql: "select id from t1 group by 2", + expErr: "Unknown column '2' in 'group clause'", }, { - sql: "select t1.col from t1 order by 'fubick'", - expSQL: "select t1.col from t1 order by 'fubick' asc", + sql: "select *, id from t1 group by 2", + expErr: "cannot use column offsets in group clause when using `*`", + }} + for _, tcase := range tcases { + t.Run(tcase.sql, func(t *testing.T) { + ast, err := sqlparser.NewTestParser().Parse(tcase.sql) + require.NoError(t, err) + selectStatement := ast.(*sqlparser.Select) + st, err := Analyze(selectStatement, cDB, schemaInfo) + if tcase.expErr == "" { + require.NoError(t, err) + assert.Equal(t, tcase.expSQL, sqlparser.String(selectStatement)) + gb := selectStatement.GroupBy + deps := st.RecursiveDeps(gb[0]) + assert.Equal(t, tcase.expDeps, deps) + } else { + require.EqualError(t, err, tcase.expErr) + } + }) + } +} + +func TestOrderByLiteral(t *testing.T) { + schemaInfo := &FakeSI{ + Tables: map[string]*vindexes.Table{}, + } + cDB := "db" + tcases := []struct { + sql string + expSQL string + expDeps TableSet + expErr string + }{{ + sql: "select 1 as id from t1 order by 1", + expSQL: "select 1 as id from t1 order by '' asc", + expDeps: NoTables, }, { - sql: "select t1.col as foo from t1 order by 1", - expSQL: "select t1.col as foo from t1 order by t1.col asc", + sql: "select t1.col from t1 order by 1", + expSQL: "select t1.col from t1 order by t1.col asc", + expDeps: TS0, }, { - sql: "select t1.col from t1 group by 1", - expSQL: "select t1.col from t1 group by t1.col", + sql: "select t1.col from t1 order by 1.0", + expSQL: "select t1.col from t1 order by 1.0 asc", + expDeps: NoTables, }, { - sql: "select t1.col as xyz from t1 group by 1", - expSQL: "select t1.col as xyz from t1 group by t1.col", + sql: "select t1.col from t1 order by 'fubick'", + expSQL: "select t1.col from t1 order by 'fubick' asc", + expDeps: NoTables, }, { - sql: "select t1.col as xyz, count(*) from t1 group by 1 order by 2", - expSQL: "select t1.col as xyz, count(*) from t1 group by t1.col order by count(*) asc", + sql: "select t1.col as foo from t1 order by 1", + expSQL: "select t1.col as foo from t1 order by t1.col asc", + expDeps: TS0, }, { - sql: "select id from t1 group by 2", - expErr: "Unknown column '2' in 'group clause'", + sql: "select t1.col as xyz, count(*) from t1 group by 1 order by 2", + expSQL: "select t1.col as xyz, count(*) from t1 group by t1.col order by count(*) asc", + expDeps: TS0, }, { sql: "select id from t1 order by 2", expErr: "Unknown column '2' in 'order clause'", @@ -347,27 +389,41 @@ func TestOrderByGroupByLiteral(t *testing.T) { sql: "select *, id from t1 order by 2", expErr: "cannot use column offsets in order clause when using `*`", }, { - sql: "select *, id from t1 group by 2", - expErr: "cannot use column offsets in group clause when using `*`", + sql: "select id from t1 order by 1 collate utf8_general_ci", + expSQL: "select id from t1 order by id collate utf8_general_ci asc", + expDeps: TS0, + }, { + sql: "select id from `user` union select 1 from dual order by 1", + expSQL: "select id from `user` union select 1 from dual order by id asc", + expDeps: TS0, }, { - sql: "select id from t1 order by 1 collate utf8_general_ci", - expSQL: "select id from t1 order by id collate utf8_general_ci asc", + sql: "select id from t1 order by 2", + expErr: "Unknown column '2' in 'order clause'", }, { - sql: "select a.id from `user` union select 1 from dual order by 1", - expSQL: "select a.id from `user` union select 1 from dual order by id asc", + sql: "select a.id, b.id from user as a, user_extra as b union select 1, 2 order by 1", + expSQL: "select a.id, b.id from `user` as a, user_extra as b union select 1, 2 from dual order by id asc", + expDeps: TS0, }, { - sql: "select a.id, b.id from user as a, user_extra as b union select 1, 2 order by 1", - expErr: "Column 'id' in field list is ambiguous", + sql: "select a.id, b.id from user as a, user_extra as b union select 1, 2 order by 2", + expSQL: "select a.id, b.id from `user` as a, user_extra as b union select 1, 2 from dual order by id asc", + expDeps: TS1, + }, { + sql: "select user.id as foo from user union select col from user_extra order by 1", + expSQL: "select `user`.id as foo from `user` union select col from user_extra order by foo asc", + expDeps: MergeTableSets(TS0, TS1), }} for _, tcase := range tcases { t.Run(tcase.sql, func(t *testing.T) { ast, err := sqlparser.NewTestParser().Parse(tcase.sql) require.NoError(t, err) selectStatement := ast.(sqlparser.SelectStatement) - _, err = Analyze(selectStatement, cDB, schemaInfo) + st, err := Analyze(selectStatement, cDB, schemaInfo) if tcase.expErr == "" { require.NoError(t, err) assert.Equal(t, tcase.expSQL, sqlparser.String(selectStatement)) + ordering := selectStatement.GetOrderBy() + deps := st.RecursiveDeps(ordering[0].Expr) + assert.Equal(t, tcase.expDeps, deps) } else { require.EqualError(t, err, tcase.expErr) } @@ -375,7 +431,7 @@ func TestOrderByGroupByLiteral(t *testing.T) { } } -func TestHavingAndOrderByColumnName(t *testing.T) { +func TestHavingColumnName(t *testing.T) { schemaInfo := &FakeSI{ Tables: map[string]*vindexes.Table{}, } @@ -388,28 +444,97 @@ func TestHavingAndOrderByColumnName(t *testing.T) { sql: "select id, sum(foo) as sumOfFoo from t1 having sumOfFoo > 1", expSQL: "select id, sum(foo) as sumOfFoo from t1 having sum(foo) > 1", }, { + sql: "select id, sum(foo) as foo from t1 having sum(foo) > 1", + expSQL: "select id, sum(foo) as foo from t1 having sum(foo) > 1", + }, { + sql: "select foo + 2 as foo from t1 having foo = 42", + expSQL: "select foo + 2 as foo from t1 having foo + 2 = 42", + }} + for _, tcase := range tcases { + t.Run(tcase.sql, func(t *testing.T) { + ast, err := sqlparser.NewTestParser().Parse(tcase.sql) + require.NoError(t, err) + selectStatement := ast.(sqlparser.SelectStatement) + _, err = Analyze(selectStatement, cDB, schemaInfo) + if tcase.expErr == "" { + require.NoError(t, err) + assert.Equal(t, tcase.expSQL, sqlparser.String(selectStatement)) + } else { + require.EqualError(t, err, tcase.expErr) + } + }) + } +} + +func TestOrderByColumnName(t *testing.T) { + schemaInfo := &FakeSI{ + Tables: map[string]*vindexes.Table{ + "t1": { + Keyspace: &vindexes.Keyspace{Name: "ks", Sharded: true}, + Name: sqlparser.NewIdentifierCS("t1"), + Columns: []vindexes.Column{{ + Name: sqlparser.NewIdentifierCI("id"), + Type: sqltypes.VarChar, + }, { + Name: sqlparser.NewIdentifierCI("foo"), + Type: sqltypes.VarChar, + }, { + Name: sqlparser.NewIdentifierCI("bar"), + Type: sqltypes.VarChar, + }}, + ColumnListAuthoritative: true, + }, + }, + } + cDB := "db" + tcases := []struct { + sql string + expSQL string + expErr string + }{{ sql: "select id, sum(foo) as sumOfFoo from t1 order by sumOfFoo", expSQL: "select id, sum(foo) as sumOfFoo from t1 order by sum(foo) asc", }, { - sql: "select id, sum(foo) as foo from t1 having sum(foo) > 1", - expSQL: "select id, sum(foo) as foo from t1 having sum(foo) > 1", + sql: "select id, sum(foo) as sumOfFoo from t1 order by sumOfFoo + 1", + expSQL: "select id, sum(foo) as sumOfFoo from t1 order by sum(foo) + 1 asc", + }, { + sql: "select id, sum(foo) as sumOfFoo from t1 order by abs(sumOfFoo)", + expSQL: "select id, sum(foo) as sumOfFoo from t1 order by abs(sum(foo)) asc", + }, { + sql: "select id, sum(foo) as sumOfFoo from t1 order by max(sumOfFoo)", + expErr: "Invalid use of group function", + }, { + sql: "select id, sum(foo) as foo from t1 order by foo + 1", + expSQL: "select id, sum(foo) as foo from t1 order by foo + 1 asc", + }, { + sql: "select id, sum(foo) as foo from t1 order by foo", + expSQL: "select id, sum(foo) as foo from t1 order by sum(foo) asc", }, { sql: "select id, lower(min(foo)) as foo from t1 order by min(foo)", expSQL: "select id, lower(min(foo)) as foo from t1 order by min(foo) asc", }, { - // invalid according to group by rules, but still accepted by mysql - sql: "select id, t1.bar as foo from t1 group by id order by min(foo)", - expSQL: "select id, t1.bar as foo from t1 group by id order by min(t1.bar) asc", + sql: "select id, lower(min(foo)) as foo from t1 order by foo", + expSQL: "select id, lower(min(foo)) as foo from t1 order by lower(min(foo)) asc", }, { - sql: "select foo + 2 as foo from t1 having foo = 42", - expSQL: "select foo + 2 as foo from t1 having foo + 2 = 42", + sql: "select id, lower(min(foo)) as foo from t1 order by abs(foo)", + expSQL: "select id, lower(min(foo)) as foo from t1 order by abs(foo) asc", }, { - sql: "select id, b as id, count(*) from t1 order by id", + sql: "select id, t1.bar as foo from t1 group by id order by min(foo)", + expSQL: "select id, t1.bar as foo from t1 group by id order by min(foo) asc", + }, { + sql: "select id, bar as id, count(*) from t1 order by id", expErr: "Column 'id' in field list is ambiguous", }, { sql: "select id, id, count(*) from t1 order by id", expSQL: "select id, id, count(*) from t1 order by id asc", - }} + }, { + sql: "select id, count(distinct foo) k from t1 group by id order by k", + expSQL: "select id, count(distinct foo) as k from t1 group by id order by count(distinct foo) asc", + }, { + sql: "select user.id as foo from user union select col from user_extra order by foo", + expSQL: "select `user`.id as foo from `user` union select col from user_extra order by foo asc", + }, + } for _, tcase := range tcases { t.Run(tcase.sql, func(t *testing.T) { ast, err := sqlparser.NewTestParser().Parse(tcase.sql) diff --git a/go/vt/vtgate/semantics/errors.go b/go/vt/vtgate/semantics/errors.go index 0be85633632..a903408ba9d 100644 --- a/go/vt/vtgate/semantics/errors.go +++ b/go/vt/vtgate/semantics/errors.go @@ -52,6 +52,7 @@ type ( SubqueryColumnCountError struct{ Expected int } ColumnsMissingInSchemaError struct{} CantUseMultipleVindexHints struct{ Table string } + InvalidUserOfGroupFunction struct{} NoSuchVindexFound struct { Table string @@ -213,18 +214,18 @@ func (e *BuggyError) Error() string { func (e *BuggyError) bug() {} // ColumnNotFoundError -func (e *ColumnNotFoundError) Error() string { +func (e ColumnNotFoundError) Error() string { if e.Table == nil { return eprintf(e, "column '%s' not found", sqlparser.String(e.Column)) } return eprintf(e, "column '%s' not found in table '%s'", sqlparser.String(e.Column), sqlparser.String(e.Table)) } -func (e *ColumnNotFoundError) ErrorCode() vtrpcpb.Code { +func (e ColumnNotFoundError) ErrorCode() vtrpcpb.Code { return vtrpcpb.Code_INVALID_ARGUMENT } -func (e *ColumnNotFoundError) ErrorState() vterrors.State { +func (e ColumnNotFoundError) ErrorState() vterrors.State { return vterrors.BadFieldError } @@ -241,6 +242,7 @@ func (e *AmbiguousColumnError) ErrorCode() vtrpcpb.Code { return vtrpcpb.Code_INVALID_ARGUMENT } +// UnsupportedConstruct func (e *UnsupportedConstruct) unsupported() {} func (e *UnsupportedConstruct) ErrorCode() vtrpcpb.Code { @@ -251,6 +253,7 @@ func (e *UnsupportedConstruct) Error() string { return eprintf(e, e.errString) } +// SubqueryColumnCountError func (e *SubqueryColumnCountError) ErrorCode() vtrpcpb.Code { return vtrpcpb.Code_INVALID_ARGUMENT } @@ -259,7 +262,7 @@ func (e *SubqueryColumnCountError) Error() string { return fmt.Sprintf("Operand should contain %d column(s)", e.Expected) } -// MissingInVSchemaError +// ColumnsMissingInSchemaError func (e *ColumnsMissingInSchemaError) Error() string { return "VT09015: schema tracking required" } @@ -277,7 +280,7 @@ func (c *CantUseMultipleVindexHints) ErrorCode() vtrpcpb.Code { return vtrpcpb.Code_FAILED_PRECONDITION } -// CantUseMultipleVindexHints +// NoSuchVindexFound func (c *NoSuchVindexFound) Error() string { return vterrors.VT09021(c.VindexName, c.Table).Error() } @@ -285,3 +288,16 @@ func (c *NoSuchVindexFound) Error() string { func (c *NoSuchVindexFound) ErrorCode() vtrpcpb.Code { return vtrpcpb.Code_FAILED_PRECONDITION } + +// InvalidUserOfGroupFunction +func (*InvalidUserOfGroupFunction) Error() string { + return "Invalid use of group function" +} + +func (*InvalidUserOfGroupFunction) ErrorCode() vtrpcpb.Code { + return vtrpcpb.Code_INVALID_ARGUMENT +} + +func (*InvalidUserOfGroupFunction) ErrorState() vterrors.State { + return vterrors.InvalidGroupFuncUse +} From ba3531f59d417d053431183d943d6ecd5f206876 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Thu, 22 Feb 2024 13:47:29 +0100 Subject: [PATCH 08/16] Revert "Skip for-loop alloc in `go/vt/discovery/healthcheck.go`" (#15328) Signed-off-by: Dirkjan Bussink --- go/vt/discovery/healthcheck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 5572e9a41d9..5d6a5e32662 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -774,7 +774,6 @@ func FilterTargetsByKeyspaces(keyspaces []string, targets []*query.Target) []*qu func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*query.Target, requireServing bool) error { targets = FilterTargetsByKeyspaces(KeyspacesToWatch, targets) - var tabletHealths []*TabletHealth for { // We nil targets as we find them. allPresent := true @@ -783,6 +782,7 @@ func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*query. continue } + var tabletHealths []*TabletHealth if requireServing { tabletHealths = hc.GetHealthyTabletStats(target) } else { From 6fec1193efc1393ab3edbab3a1acd56647e7d528 Mon Sep 17 00:00:00 2001 From: Manik Rana Date: Thu, 22 Feb 2024 19:19:15 +0530 Subject: [PATCH 09/16] chore: modernize tests (#15244) Signed-off-by: Manik Rana Signed-off-by: Manik Rana --- go/cache/lru_cache_test.go | 34 ++---- go/cache/theine/singleflight_test.go | 14 ++- go/event/event_test.go | 40 ++---- go/event/syslogger/syslogger_test.go | 63 ++++------ go/flagutil/flagutil_test.go | 10 +- go/history/history_test.go | 12 +- go/mysql/decimal/decimal_test.go | 174 +++++++++------------------ go/streamlog/streamlog_test.go | 2 - go/trace/trace_test.go | 1 - 9 files changed, 116 insertions(+), 234 deletions(-) diff --git a/go/cache/lru_cache_test.go b/go/cache/lru_cache_test.go index e0dbfcda117..af9db72852e 100644 --- a/go/cache/lru_cache_test.go +++ b/go/cache/lru_cache_test.go @@ -18,6 +18,8 @@ package cache import ( "testing" + + "github.com/stretchr/testify/assert" ) type CacheValue struct { @@ -27,24 +29,12 @@ type CacheValue struct { func TestInitialState(t *testing.T) { cache := NewLRUCache[*CacheValue](5) l, sz, c, e, h, m := cache.Len(), cache.UsedCapacity(), cache.MaxCapacity(), cache.Evictions(), cache.Hits(), cache.Misses() - if l != 0 { - t.Errorf("length = %v, want 0", l) - } - if sz != 0 { - t.Errorf("size = %v, want 0", sz) - } - if c != 5 { - t.Errorf("capacity = %v, want 5", c) - } - if e != 0 { - t.Errorf("evictions = %v, want 0", c) - } - if h != 0 { - t.Errorf("hits = %v, want 0", c) - } - if m != 0 { - t.Errorf("misses = %v, want 0", c) - } + assert.Zero(t, l) + assert.EqualValues(t, 0, sz) + assert.EqualValues(t, 5, c) + assert.EqualValues(t, 0, e) + assert.EqualValues(t, 0, h) + assert.EqualValues(t, 0, m) } func TestSetInsertsValue(t *testing.T) { @@ -137,12 +127,8 @@ func TestCapacityIsObeyed(t *testing.T) { // Insert one more; something should be evicted to make room. cache.Set("key4", value) sz, evictions := cache.UsedCapacity(), cache.Evictions() - if sz != size { - t.Errorf("post-evict cache.UsedCapacity() = %v, expected %v", sz, size) - } - if evictions != 1 { - t.Errorf("post-evict cache.Evictions() = %v, expected 1", evictions) - } + assert.Equal(t, size, sz) + assert.EqualValues(t, 1, evictions) // Check various other stats if l := cache.Len(); int64(l) != size { diff --git a/go/cache/theine/singleflight_test.go b/go/cache/theine/singleflight_test.go index 4ae235f97e2..bf5018a8891 100644 --- a/go/cache/theine/singleflight_test.go +++ b/go/cache/theine/singleflight_test.go @@ -43,8 +43,8 @@ func TestDo(t *testing.T) { return "bar", nil }) - assert.Equal(t, "bar (string)", fmt.Sprintf("%v (%T)", v, v), "incorrect Do value") - assert.NoError(t, err, "got Do error") + assert.Equal(t, "bar (string)", fmt.Sprintf("%v (%T)", v, v)) + assert.NoError(t, err) } func TestDoErr(t *testing.T) { @@ -85,11 +85,11 @@ func TestDoDupSuppress(t *testing.T) { defer wg2.Done() wg1.Done() v, err, _ := g.Do("key", fn) - if !assert.NoError(t, err, "unexpected Do error") { + if !assert.NoError(t, err) { return } - assert.Equal(t, "bar", v, "unexpected Do value") + assert.Equal(t, "bar", v) }() } wg1.Wait() @@ -98,7 +98,8 @@ func TestDoDupSuppress(t *testing.T) { c <- "bar" wg2.Wait() got := atomic.LoadInt32(&calls) - assert.True(t, got > 0 && got < n, "number of calls not between 0 and %d", n) + assert.Greater(t, got, int32(0)) + assert.Less(t, got, int32(n)) } // Test singleflight behaves correctly after Do panic. @@ -131,7 +132,7 @@ func TestPanicDo(t *testing.T) { select { case <-done: - assert.Equal(t, int32(n), panicCount, "unexpected number of panics") + assert.EqualValues(t, n, panicCount) case <-time.After(time.Second): require.Fail(t, "Do hangs") } @@ -152,6 +153,7 @@ func TestGoexitDo(t *testing.T) { var err error defer func() { assert.NoError(t, err) + if atomic.AddInt32(&waited, -1) == 0 { close(done) } diff --git a/go/event/event_test.go b/go/event/event_test.go index f7356a98f3e..cdca98abd85 100644 --- a/go/event/event_test.go +++ b/go/event/event_test.go @@ -20,6 +20,8 @@ import ( "reflect" "testing" "time" + + "github.com/stretchr/testify/assert" ) type testInterface1 interface { @@ -56,10 +58,7 @@ func TestStaticListener(t *testing.T) { AddListener(func(testEvent1) { triggered = true }) AddListener(func(testEvent2) { t.Errorf("wrong listener type triggered") }) Dispatch(testEvent1{}) - - if !triggered { - t.Errorf("static listener failed to trigger") - } + assert.True(t, triggered, "static listener failed to trigger") } func TestPointerListener(t *testing.T) { @@ -69,10 +68,7 @@ func TestPointerListener(t *testing.T) { AddListener(func(ev *testEvent2) { ev.triggered = true }) AddListener(func(testEvent2) { t.Errorf("non-pointer listener triggered on pointer type") }) Dispatch(testEvent) - - if !testEvent.triggered { - t.Errorf("pointer listener failed to trigger") - } + assert.True(t, testEvent.triggered, "pointer listener failed to trigger") } func TestInterfaceListener(t *testing.T) { @@ -82,10 +78,7 @@ func TestInterfaceListener(t *testing.T) { AddListener(func(testInterface1) { triggered = true }) AddListener(func(testInterface2) { t.Errorf("interface listener triggered on non-matching type") }) Dispatch(testEvent1{}) - - if !triggered { - t.Errorf("interface listener failed to trigger") - } + assert.True(t, triggered, "interface listener failed to trigger") } func TestEmptyInterfaceListener(t *testing.T) { @@ -94,10 +87,7 @@ func TestEmptyInterfaceListener(t *testing.T) { triggered := false AddListener(func(any) { triggered = true }) Dispatch("this should match any") - - if !triggered { - t.Errorf("any listener failed to trigger") - } + assert.True(t, triggered, "empty listener failed to trigger") } func TestMultipleListeners(t *testing.T) { @@ -144,7 +134,6 @@ func TestBadListenerWrongType(t *testing.T) { defer func() { err := recover() - if err == nil { t.Errorf("bad listener type (not a func) failed to trigger panic") } @@ -186,10 +175,8 @@ func TestDispatchPointerToValueInterfaceListener(t *testing.T) { triggered = true }) Dispatch(&testEvent1{}) + assert.True(t, triggered, "Dispatch by pointer failed to trigger interface listener") - if !triggered { - t.Errorf("Dispatch by pointer failed to trigger interface listener") - } } func TestDispatchValueToValueInterfaceListener(t *testing.T) { @@ -200,10 +187,7 @@ func TestDispatchValueToValueInterfaceListener(t *testing.T) { triggered = true }) Dispatch(testEvent1{}) - - if !triggered { - t.Errorf("Dispatch by value failed to trigger interface listener") - } + assert.True(t, triggered, "Dispatch by value failed to trigger interface listener") } func TestDispatchPointerToPointerInterfaceListener(t *testing.T) { @@ -212,10 +196,8 @@ func TestDispatchPointerToPointerInterfaceListener(t *testing.T) { triggered := false AddListener(func(testInterface2) { triggered = true }) Dispatch(&testEvent2{}) + assert.True(t, triggered, "interface listener failed to trigger for pointer") - if !triggered { - t.Errorf("interface listener failed to trigger for pointer") - } } func TestDispatchValueToPointerInterfaceListener(t *testing.T) { @@ -245,10 +227,8 @@ func TestDispatchUpdate(t *testing.T) { ev := &testUpdateEvent{} DispatchUpdate(ev, "hello") + assert.True(t, triggered, "listener failed to trigger on DispatchUpdate()") - if !triggered { - t.Errorf("listener failed to trigger on DispatchUpdate()") - } want := "hello" if got := ev.update.(string); got != want { t.Errorf("ev.update = %#v, want %#v", got, want) diff --git a/go/event/syslogger/syslogger_test.go b/go/event/syslogger/syslogger_test.go index 28925d2ac52..6c6a181d2e5 100644 --- a/go/event/syslogger/syslogger_test.go +++ b/go/event/syslogger/syslogger_test.go @@ -25,6 +25,8 @@ import ( "testing" "vitess.io/vitess/go/event" + + "github.com/stretchr/testify/assert" ) type TestEvent struct { @@ -70,10 +72,8 @@ func TestSyslog(t *testing.T) { ev := new(TestEvent) event.Dispatch(ev) + assert.True(t, ev.triggered) - if !ev.triggered { - t.Errorf("Syslog() was not called on event that implements Syslogger") - } } // TestBadWriter verifies we are still triggering (to normal logs) if @@ -87,55 +87,40 @@ func TestBadWriter(t *testing.T) { wantLevel := "ERROR" ev := &TestEvent{priority: syslog.LOG_ALERT, message: wantMsg} event.Dispatch(ev) - if !strings.Contains(tl.getLog().msg, wantMsg) { - t.Errorf("error log msg [%s], want msg [%s]", tl.getLog().msg, wantMsg) - } - if !strings.Contains(tl.getLog().level, wantLevel) { - t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) - } + assert.True(t, strings.Contains(tl.getLog().msg, wantMsg)) + assert.True(t, strings.Contains(tl.getLog().level, wantLevel)) + ev = &TestEvent{priority: syslog.LOG_CRIT, message: wantMsg} event.Dispatch(ev) - if !strings.Contains(tl.getLog().level, wantLevel) { - t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) - } + assert.True(t, strings.Contains(tl.getLog().level, wantLevel)) + ev = &TestEvent{priority: syslog.LOG_ERR, message: wantMsg} event.Dispatch(ev) - if !strings.Contains(tl.getLog().level, wantLevel) { - t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) - } + assert.True(t, strings.Contains(tl.getLog().level, wantLevel)) + ev = &TestEvent{priority: syslog.LOG_EMERG, message: wantMsg} event.Dispatch(ev) - if !strings.Contains(tl.getLog().level, wantLevel) { - t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) - } + assert.True(t, strings.Contains(tl.getLog().level, wantLevel)) wantLevel = "WARNING" ev = &TestEvent{priority: syslog.LOG_WARNING, message: wantMsg} event.Dispatch(ev) - if !strings.Contains(tl.getLog().level, wantLevel) { - t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) - } + assert.True(t, strings.Contains(tl.getLog().level, wantLevel)) wantLevel = "INFO" ev = &TestEvent{priority: syslog.LOG_INFO, message: wantMsg} event.Dispatch(ev) - if !strings.Contains(tl.getLog().level, wantLevel) { - t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) - } + assert.True(t, strings.Contains(tl.getLog().level, wantLevel)) + ev = &TestEvent{priority: syslog.LOG_NOTICE, message: wantMsg} event.Dispatch(ev) - if !strings.Contains(tl.getLog().level, wantLevel) { - t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) - } + assert.True(t, strings.Contains(tl.getLog().level, wantLevel)) + ev = &TestEvent{priority: syslog.LOG_DEBUG, message: wantMsg} event.Dispatch(ev) - if !strings.Contains(tl.getLog().level, wantLevel) { - t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) - } + assert.True(t, strings.Contains(tl.getLog().level, wantLevel)) + assert.True(t, ev.triggered) - if !ev.triggered { - t.Errorf("passed nil writer to client") - } } // TestWriteError checks that we don't panic on a write error. @@ -150,10 +135,8 @@ func TestInvalidSeverity(t *testing.T) { writer = fw event.Dispatch(&TestEvent{priority: syslog.Priority(123), message: "log me"}) + assert.NotEqual(t, "log me", fw.message) - if fw.message == "log me" { - t.Errorf("message was logged despite invalid severity") - } } func testSeverity(sev syslog.Priority, t *testing.T) { @@ -161,13 +144,9 @@ func testSeverity(sev syslog.Priority, t *testing.T) { writer = fw event.Dispatch(&TestEvent{priority: sev, message: "log me"}) + assert.Equal(t, sev, fw.priority) + assert.Equal(t, "log me", fw.message) - if fw.priority != sev { - t.Errorf("wrong priority: got %v, want %v", fw.priority, sev) - } - if fw.message != "log me" { - t.Errorf(`wrong message: got "%v", want "%v"`, fw.message, "log me") - } } func TestEmerg(t *testing.T) { diff --git a/go/flagutil/flagutil_test.go b/go/flagutil/flagutil_test.go index dda17bb13d7..2502213ab73 100644 --- a/go/flagutil/flagutil_test.go +++ b/go/flagutil/flagutil_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -38,12 +39,9 @@ func TestStringList(t *testing.T) { t.Errorf("v.Set(%v): %v", in, err) continue } - if strings.Join(p, ".") != out { - t.Errorf("want %#v, got %#v", strings.Split(out, "."), p) - } - if p.String() != in { - t.Errorf("v.String(): want %#v, got %#v", in, p.String()) - } + assert.Equal(t, out, strings.Join(p, ".")) + assert.Equal(t, in, p.String()) + } } diff --git a/go/history/history_test.go b/go/history/history_test.go index 3e817b34bb2..34c57756315 100644 --- a/go/history/history_test.go +++ b/go/history/history_test.go @@ -18,6 +18,8 @@ package history import ( "testing" + + "github.com/stretchr/testify/assert" ) func TestHistory(t *testing.T) { @@ -33,9 +35,8 @@ func TestHistory(t *testing.T) { t.Errorf("len(records): want %v, got %v. records: %+v", want, got, q) } for i, record := range records { - if record != want[i] { - t.Errorf("record doesn't match: want %v, got %v", want[i], record) - } + assert.Equal(t, want[i], record) + } for ; i < 6; i++ { @@ -48,9 +49,8 @@ func TestHistory(t *testing.T) { t.Errorf("len(records): want %v, got %v. records: %+v", want, got, q) } for i, record := range records { - if record != want[i] { - t.Errorf("record doesn't match: want %v, got %v", want[i], record) - } + assert.Equal(t, want[i], record) + } } diff --git a/go/mysql/decimal/decimal_test.go b/go/mysql/decimal/decimal_test.go index 09819ddcebb..3499ab8f802 100644 --- a/go/mysql/decimal/decimal_test.go +++ b/go/mysql/decimal/decimal_test.go @@ -26,6 +26,8 @@ import ( "strings" "testing" "testing/quick" + + "github.com/stretchr/testify/assert" ) type testEnt struct { @@ -120,11 +122,8 @@ func TestNewFromFloat(t *testing.T) { for _, x := range testTable { s := x.short d := NewFromFloat(x.float) - if d.String() != s { - t.Errorf("expected %s, got %s (float: %v) (%s, %d)", - s, d.String(), x.float, - d.value.String(), d.exp) - } + assert.Equal(t, s, d.String()) + } shouldPanicOn := []float64{ @@ -156,11 +155,8 @@ func TestNewFromFloatRandom(t *testing.T) { continue } got := NewFromFloat(in) - if !want.Equal(got) { - t.Errorf("in: %v, expected %s (%s, %d), got %s (%s, %d) ", - in, want.String(), want.value.String(), want.exp, - got.String(), got.value.String(), got.exp) - } + assert.True(t, want.Equal(got)) + } } @@ -193,11 +189,8 @@ func TestNewFromFloat32Random(t *testing.T) { continue } got := NewFromFloat32(in) - if !want.Equal(got) { - t.Errorf("in: %v, expected %s (%s, %d), got %s (%s, %d) ", - in, want.String(), want.value.String(), want.exp, - got.String(), got.value.String(), got.exp) - } + assert.True(t, want.Equal(got)) + } } @@ -317,14 +310,9 @@ func TestNewFromStringErrs(t *testing.T) { for s, o := range tests { out, err := NewFromString(s) + assert.Error(t, err) + assert.Equal(t, o, out.String()) - if err == nil { - t.Errorf("error expected when parsing %s", s) - } - - if out.String() != o { - t.Errorf("expected %s, got %s", o, out.String()) - } } } @@ -353,11 +341,8 @@ func TestNewFromStringDeepEquals(t *testing.T) { if err1 != nil || err2 != nil { t.Errorf("error parsing strings to decimals") } + assert.Equal(t, cmp.expected, reflect.DeepEqual(d1, d2)) - if reflect.DeepEqual(d1, d2) != cmp.expected { - t.Errorf("comparison result is different from expected results for %s and %s", - cmp.str1, cmp.str2) - } } } @@ -413,11 +398,8 @@ func TestNewFromInt(t *testing.T) { for input, s := range tests { d := NewFromInt(input) - if d.String() != s { - t.Errorf("expected %s, got %s (%s, %d)", - s, d.String(), - d.value.String(), d.exp) - } + assert.Equal(t, s, d.String()) + } } @@ -499,20 +481,15 @@ func TestDecimal_RoundAndStringFixed(t *testing.T) { t.Fatal(err) } got := d.Round(test.places) - if !got.Equal(expected) { - t.Errorf("Rounding %s to %d places, got %s, expected %s", - d, test.places, got, expected) - } + assert.True(t, got.Equal(expected)) // test StringFixed if test.expectedFixed == "" { test.expectedFixed = test.expected } gotStr := d.StringFixed(test.places) - if gotStr != test.expectedFixed { - t.Errorf("(%s).StringFixed(%d): got %s, expected %s", - d, test.places, gotStr, test.expectedFixed) - } + assert.Equal(t, test.expectedFixed, gotStr) + } } @@ -541,9 +518,8 @@ func TestDecimal_Add(t *testing.T) { t.FailNow() } c := a.Add(b) - if c.String() != res { - t.Errorf("expected %s, got %s", res, c.String()) - } + assert.Equal(t, res, c.String()) + } } @@ -576,9 +552,8 @@ func TestDecimal_Sub(t *testing.T) { t.FailNow() } c := a.sub(b) - if c.String() != res { - t.Errorf("expected %s, got %s", res, c.String()) - } + assert.Equal(t, res, c.String()) + } } @@ -597,18 +572,16 @@ func TestDecimal_Neg(t *testing.T) { t.FailNow() } b := a.Neg() - if b.String() != res { - t.Errorf("expected %s, got %s", res, b.String()) - } + assert.Equal(t, res, b.String()) + } } func TestDecimal_NegFromEmpty(t *testing.T) { a := Decimal{} b := a.Neg() - if b.String() != "0" { - t.Errorf("expected %s, got %s", "0", b) - } + assert.Equal(t, "0", b.String()) + } func TestDecimal_Mul(t *testing.T) { @@ -635,16 +608,14 @@ func TestDecimal_Mul(t *testing.T) { t.FailNow() } c := a.mul(b) - if c.String() != res { - t.Errorf("expected %s, got %s", res, c.String()) - } + assert.Equal(t, res, c.String()) + } // positive scale c := New(1234, 5).mul(New(45, -1)) - if c.String() != "555300000" { - t.Errorf("Expected %s, got %s", "555300000", c.String()) - } + assert.Equal(t, "555300000", c.String()) + } func TestDecimal_Div(t *testing.T) { @@ -679,14 +650,11 @@ func TestDecimal_Div(t *testing.T) { } got := num.div(denom) expected, _ := NewFromString(expectedStr) - if !got.Equal(expected) { - t.Errorf("expected %v when dividing %v by %v, got %v", - expected, num, denom, got) - } + assert.True(t, got.Equal(expected)) + got2 := num.divRound(denom, int32(divisionPrecision)) - if !got2.Equal(expected) { - t.Errorf("expected %v on divRound (%v,%v), got %v", expected, num, denom, got2) - } + assert.True(t, got2.Equal(expected)) + } type Inp2 struct { @@ -717,10 +685,8 @@ func TestDecimal_Div(t *testing.T) { expected = "-" + expectedAbs } got := num.div(denom) - if got.String() != expected { - t.Errorf("expected %s when dividing %v by %v, got %v", - expected, num, denom, got) - } + assert.Equal(t, expected, got.String()) + } } } @@ -761,14 +727,8 @@ func TestDecimal_QuoRem(t *testing.T) { t.Errorf("bad QuoRem division %s , %s , %d got %v, %v expected %s , %s", inp4.d, inp4.d2, prec, q, r, inp4.q, inp4.r) } - if !d.Equal(d2.mul(q).Add(r)) { - t.Errorf("not fitting: d=%v, d2= %v, prec=%d, q=%v, r=%v", - d, d2, prec, q, r) - } - if !q.Equal(q.Truncate(prec)) { - t.Errorf("quotient wrong precision: d=%v, d2= %v, prec=%d, q=%v, r=%v", - d, d2, prec, q, r) - } + assert.True(t, d.Equal(d2.mul(q).Add(r))) + assert.True(t, q.Equal(q.Truncate(prec))) if r.Abs().Cmp(d2.Abs().mul(New(1, -prec))) >= 0 { t.Errorf("remainder too large: d=%v, d2= %v, prec=%d, q=%v, r=%v", d, d2, prec, q, r) @@ -823,15 +783,10 @@ func TestDecimal_QuoRem2(t *testing.T) { prec := tc.prec q, r := d.QuoRem(d2, prec) // rule 1: d = d2*q +r - if !d.Equal(d2.mul(q).Add(r)) { - t.Errorf("not fitting, d=%v, d2=%v, prec=%d, q=%v, r=%v", - d, d2, prec, q, r) - } + assert.True(t, d.Equal(d2.mul(q).Add(r))) // rule 2: q is integral multiple of 10^(-prec) - if !q.Equal(q.Truncate(prec)) { - t.Errorf("quotient wrong precision, d=%v, d2=%v, prec=%d, q=%v, r=%v", - d, d2, prec, q, r) - } + assert.True(t, q.Equal(q.Truncate(prec))) + // rule 3: abs(r)= 0 { t.Errorf("remainder too large, d=%v, d2=%v, prec=%d, q=%v, r=%v", @@ -894,9 +849,8 @@ func TestDecimal_DivRound(t *testing.T) { if x.Cmp(d2.Abs().mul(New(-1, -prec))) <= 0 { t.Errorf("wrong rounding, got: %v/%v prec=%d is about %v", d, d2, prec, q) } - if !q.Equal(result) { - t.Errorf("rounded division wrong %s / %s scale %d = %s, got %v", s.d, s.d2, prec, s.result, q) - } + assert.True(t, q.Equal(result)) + } } @@ -953,9 +907,8 @@ func TestDecimal_Mod(t *testing.T) { t.FailNow() } c := a.mod(b) - if c.String() != res { - t.Errorf("expected %s, got %s", res, c.String()) - } + assert.Equal(t, res, c.String()) + } } @@ -972,9 +925,8 @@ func TestDecimal_Overflow(t *testing.T) { func TestDecimal_Scale(t *testing.T) { a := New(1234, -3) - if a.Exponent() != -3 { - t.Errorf("error") - } + assert.EqualValues(t, -3, a.Exponent()) + } func TestDecimal_Abs1(t *testing.T) { @@ -982,9 +934,8 @@ func TestDecimal_Abs1(t *testing.T) { b := New(1234, -4) c := a.Abs() - if c.Cmp(b) != 0 { - t.Errorf("error") - } + assert.Zero(t, c.Cmp(b)) + } func TestDecimal_Abs2(t *testing.T) { @@ -992,9 +943,8 @@ func TestDecimal_Abs2(t *testing.T) { b := New(1234, -4) c := b.Abs() - if c.Cmp(a) == 0 { - t.Errorf("error") - } + assert.NotZero(t, c.Cmp(a)) + } func TestDecimal_ScalesNotEqual(t *testing.T) { @@ -1008,19 +958,15 @@ func TestDecimal_ScalesNotEqual(t *testing.T) { func TestDecimal_Cmp1(t *testing.T) { a := New(123, 3) b := New(-1234, 2) + assert.Equal(t, 1, a.Cmp(b)) - if a.Cmp(b) != 1 { - t.Errorf("Error") - } } func TestDecimal_Cmp2(t *testing.T) { a := New(123, 3) b := New(1234, 2) + assert.Equal(t, -1, a.Cmp(b)) - if a.Cmp(b) != -1 { - t.Errorf("Error") - } } func TestDecimal_IsInteger(t *testing.T) { @@ -1045,26 +991,20 @@ func TestDecimal_IsInteger(t *testing.T) { if err != nil { t.Fatal(err) } - if d.isInteger() != testCase.IsInteger { - t.Errorf("expect %t, got %t, for %s", testCase.IsInteger, d.isInteger(), testCase.Dec) - } + assert.Equal(t, testCase.IsInteger, d.isInteger()) + } } func TestDecimal_Sign(t *testing.T) { - if Zero.Sign() != 0 { - t.Errorf("%q should have sign 0", Zero) - } + assert.Zero(t, Zero.Sign()) one := New(1, 0) - if one.Sign() != 1 { - t.Errorf("%q should have sign 1", one) - } + assert.Equal(t, 1, one.Sign()) mone := New(-1, 0) - if mone.Sign() != -1 { - t.Errorf("%q should have sign -1", mone) - } + assert.Equal(t, -1, mone.Sign()) + } func didPanic(f func()) bool { diff --git a/go/streamlog/streamlog_test.go b/go/streamlog/streamlog_test.go index e653de74ea8..ab01dd4ccb2 100644 --- a/go/streamlog/streamlog_test.go +++ b/go/streamlog/streamlog_test.go @@ -315,7 +315,6 @@ func TestShouldEmitLog(t *testing.T) { } for _, tt := range tests { - tt := tt t.Run(tt.sql, func(t *testing.T) { SetQueryLogFilterTag(tt.qLogFilterTag) SetQueryLogRowThreshold(tt.qLogRowThreshold) @@ -360,7 +359,6 @@ func TestGetFormatter(t *testing.T) { } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { var buffer bytes.Buffer logFormatterFunc := GetFormatter[string](tt.logger) diff --git a/go/trace/trace_test.go b/go/trace/trace_test.go index 6b5bdf491bb..7f1f6d8c528 100644 --- a/go/trace/trace_test.go +++ b/go/trace/trace_test.go @@ -96,7 +96,6 @@ func TestNewFromString(t *testing.T) { }, } for _, tt := range tests { - tt := tt t.Run(tt.label, func(t *testing.T) { span, ctx, err := NewFromString(context.Background(), tt.parent, tt.label) if tt.expectedErr == "" { From 89bc131fb2e64a191da221aa317d9d7cecdbf0a7 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 22 Feb 2024 09:34:18 -0500 Subject: [PATCH 10/16] delete TestActionAndTimeout (#15322) Signed-off-by: Andrew Mason --- go/test/endtoend/tabletmanager/commands_test.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/go/test/endtoend/tabletmanager/commands_test.go b/go/test/endtoend/tabletmanager/commands_test.go index 537a3b9d0fc..d23413e0269 100644 --- a/go/test/endtoend/tabletmanager/commands_test.go +++ b/go/test/endtoend/tabletmanager/commands_test.go @@ -22,7 +22,6 @@ import ( "fmt" "reflect" "testing" - "time" "vitess.io/vitess/go/test/endtoend/utils" @@ -140,18 +139,6 @@ func assertExecuteFetch(t *testing.T, qr string) { assert.Equal(t, want, got) } -// ActionAndTimeout test -func TestActionAndTimeout(t *testing.T) { - defer cluster.PanicHandler(t) - err := clusterInstance.VtctldClientProcess.ExecuteCommand("SleepTablet", primaryTablet.Alias, "5s") - require.Nil(t, err) - time.Sleep(1 * time.Second) - - // try a frontend RefreshState that should timeout as the tablet is busy running the other one - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RefreshState", primaryTablet.Alias, "--wait_timeout", "2s") - assert.Error(t, err, "timeout as tablet is in Sleep") -} - func TestHook(t *testing.T) { // test a regular program works defer cluster.PanicHandler(t) From a78ccbf716ce46bf218861ddb7e70aad5fe6ef19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Taylor?= Date: Thu, 22 Feb 2024 16:09:34 +0100 Subject: [PATCH 11/16] test: skip should mark the correct *testing.T (#15333) --- go/test/endtoend/utils/cmp.go | 6 ++++++ .../queries/aggregation/aggregation_test.go | 18 +++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/go/test/endtoend/utils/cmp.go b/go/test/endtoend/utils/cmp.go index 38726d6c3aa..8d0b56ac6b3 100644 --- a/go/test/endtoend/utils/cmp.go +++ b/go/test/endtoend/utils/cmp.go @@ -70,6 +70,12 @@ func (mcmp *MySQLCompare) AssertMatches(query, expected string) { } } +// SkipIfBinaryIsBelowVersion should be used instead of using utils.SkipIfBinaryIsBelowVersion(t, +// This is because we might be inside a Run block that has a different `t` variable +func (mcmp *MySQLCompare) SkipIfBinaryIsBelowVersion(majorVersion int, binary string) { + SkipIfBinaryIsBelowVersion(mcmp.t, majorVersion, binary) +} + // AssertMatchesAny ensures the given query produces any one of the expected results. func (mcmp *MySQLCompare) AssertMatchesAny(query string, expected ...string) { mcmp.t.Helper() diff --git a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go index 6f4dd01d4e2..da2e14218fe 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go @@ -74,7 +74,7 @@ func TestAggregateTypes(t *testing.T) { mcmp.AssertMatches("select val1 as a, count(*) from aggr_test group by a order by 2, a", `[[VARCHAR("b") INT64(1)] [VARCHAR("d") INT64(1)] [VARCHAR("a") INT64(2)] [VARCHAR("c") INT64(2)] [VARCHAR("e") INT64(2)]]`) mcmp.AssertMatches("select sum(val1) from aggr_test", `[[FLOAT64(0)]]`) mcmp.Run("Average for sharded keyspaces", func(mcmp *utils.MySQLCompare) { - utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") + mcmp.SkipIfBinaryIsBelowVersion(19, "vtgate") mcmp.AssertMatches("select avg(val1) from aggr_test", `[[FLOAT64(0)]]`) }) } @@ -178,7 +178,7 @@ func TestAggrOnJoin(t *testing.T) { `[[VARCHAR("a")]]`) mcmp.Run("Average in join for sharded", func(mcmp *utils.MySQLCompare) { - utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") + mcmp.SkipIfBinaryIsBelowVersion(19, "vtgate") mcmp.AssertMatches(`select avg(a1.val2), avg(a2.val2) from aggr_test a1 join aggr_test a2 on a1.val2 = a2.id join t3 t on a2.val2 = t.id7`, "[[DECIMAL(1.5000) DECIMAL(1.0000)]]") @@ -336,7 +336,7 @@ func TestAggOnTopOfLimit(t *testing.T) { mcmp.AssertMatches("select val1, count(*) from (select id, val1 from aggr_test where val2 < 4 order by val1 limit 2) as x group by val1", `[[NULL INT64(1)] [VARCHAR("a") INT64(1)]]`) mcmp.AssertMatchesNoOrder("select val1, count(val2) from (select val1, val2 from aggr_test limit 8) as x group by val1", `[[NULL INT64(1)] [VARCHAR("a") INT64(2)] [VARCHAR("b") INT64(1)] [VARCHAR("c") INT64(2)]]`) mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { - utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") + mcmp.SkipIfBinaryIsBelowVersion(19, "vtgate") mcmp.AssertMatches("select avg(val2) from (select id, val2 from aggr_test where val2 is null limit 2) as x", "[[NULL]]") mcmp.AssertMatchesNoOrder("select val1, avg(val2) from (select val1, val2 from aggr_test limit 8) as x group by val1", `[[NULL DECIMAL(2.0000)] [VARCHAR("a") DECIMAL(3.5000)] [VARCHAR("b") DECIMAL(1.0000)] [VARCHAR("c") DECIMAL(3.5000)]]`) }) @@ -348,7 +348,7 @@ func TestAggOnTopOfLimit(t *testing.T) { mcmp.AssertMatches("select count(val2), sum(val2) from (select id, val2 from aggr_test where val2 is null limit 2) as x", "[[INT64(0) NULL]]") mcmp.AssertMatches("select val1, count(*), sum(id) from (select id, val1 from aggr_test where val2 < 4 order by val1 limit 2) as x group by val1", `[[NULL INT64(1) DECIMAL(7)] [VARCHAR("a") INT64(1) DECIMAL(2)]]`) mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { - utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") + mcmp.SkipIfBinaryIsBelowVersion(19, "vtgate") mcmp.AssertMatches("select count(*), sum(val1), avg(val1) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2) FLOAT64(0) FLOAT64(0)]]") mcmp.AssertMatches("select count(val1), sum(id), avg(id) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2) DECIMAL(7) DECIMAL(3.5000)]]") mcmp.AssertMatchesNoOrder("select val1, count(val2), sum(val2), avg(val2) from (select val1, val2 from aggr_test limit 8) as x group by val1", @@ -370,7 +370,7 @@ func TestEmptyTableAggr(t *testing.T) { mcmp.AssertMatches(" select t1.`name`, count(*) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo' group by t1.`name`", "[]") mcmp.AssertMatches(" select t1.`name`, count(*) from t1 inner join t2 on (t1.t1_id = t2.id) where t1.value = 'foo' group by t1.`name`", "[]") mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { - utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") + mcmp.SkipIfBinaryIsBelowVersion(19, "vtgate") mcmp.AssertMatches(" select count(t1.value) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]") mcmp.AssertMatches(" select avg(t1.value) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[NULL]]") }) @@ -386,7 +386,7 @@ func TestEmptyTableAggr(t *testing.T) { mcmp.AssertMatches(" select count(*) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]") mcmp.AssertMatches(" select t1.`name`, count(*) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo' group by t1.`name`", "[]") mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { - utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") + mcmp.SkipIfBinaryIsBelowVersion(19, "vtgate") mcmp.AssertMatches(" select count(t1.value) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]") mcmp.AssertMatches(" select avg(t1.value) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[NULL]]") mcmp.AssertMatches(" select t1.`name`, count(*) from t1 inner join t2 on (t1.t1_id = t2.id) where t1.value = 'foo' group by t1.`name`", "[]") @@ -435,7 +435,7 @@ func TestAggregateLeftJoin(t *testing.T) { mcmp.AssertMatches("SELECT count(*) FROM t2 LEFT JOIN t1 ON t1.t1_id = t2.id WHERE IFNULL(t1.name, 'NOTSET') = 'r'", `[[INT64(1)]]`) mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { - utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") + mcmp.SkipIfBinaryIsBelowVersion(19, "vtgate") mcmp.AssertMatches("SELECT avg(t1.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(0.5000)]]`) mcmp.AssertMatches("SELECT avg(t2.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(1.0000)]]`) aggregations := []string{ @@ -492,7 +492,7 @@ func TestScalarAggregate(t *testing.T) { mcmp.Exec("insert into aggr_test(id, val1, val2) values(1,'a',1), (2,'A',1), (3,'b',1), (4,'c',3), (5,'c',4)") mcmp.AssertMatches("select count(distinct val1) from aggr_test", `[[INT64(3)]]`) mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { - utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") + mcmp.SkipIfBinaryIsBelowVersion(19, "vtgate") mcmp.AssertMatches("select avg(val1) from aggr_test", `[[FLOAT64(0)]]`) }) } @@ -552,7 +552,7 @@ func TestComplexAggregation(t *testing.T) { mcmp.Exec(`SELECT name+COUNT(t1_id)+1 FROM t1 GROUP BY name`) mcmp.Exec(`SELECT COUNT(*)+shardkey+MIN(t1_id)+1+MAX(t1_id)*SUM(t1_id)+1+name FROM t1 GROUP BY shardkey, name`) mcmp.Run("Average in sharded query", func(mcmp *utils.MySQLCompare) { - utils.SkipIfBinaryIsBelowVersion(t, 19, "vtgate") + mcmp.SkipIfBinaryIsBelowVersion(19, "vtgate") mcmp.Exec(`SELECT COUNT(t1_id)+MAX(shardkey)+AVG(t1_id) FROM t1`) }) } From 229bbaa7f6e2e5eb5589bf7c3ccfb3cc3732d6cd Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Fri, 23 Feb 2024 14:51:49 +0100 Subject: [PATCH 12/16] srvtopo: Setup metrics in init() function (#15304) Signed-off-by: Dirkjan Bussink --- go/cmd/vtcombo/cli/main.go | 10 +++-- go/cmd/vtexplain/cli/vtexplain.go | 4 +- go/cmd/vtgate/cli/cli.go | 9 ++++- go/cmd/vttablet/cli/cli.go | 13 ++++-- go/vt/srvtopo/discover_test.go | 4 +- go/vt/srvtopo/resilient_server.go | 10 +---- go/vt/srvtopo/resilient_server_test.go | 31 ++++++++------ go/vt/srvtopo/resolver_test.go | 8 ++-- go/vt/vtadmin/api.go | 4 +- go/vt/vtcombo/tablet_map.go | 14 ++++--- go/vt/vtexplain/vtexplain.go | 5 ++- go/vt/vtexplain/vtexplain_test.go | 7 +++- go/vt/vtexplain/vtexplain_vtgate.go | 9 +++-- go/vt/vtexplain/vtexplain_vttablet.go | 5 ++- go/vt/vtexplain/vtexplain_vttablet_test.go | 8 ++-- go/vt/vttablet/endtoend/framework/server.go | 4 +- .../tabletserver/query_executor_test.go | 4 +- go/vt/vttablet/tabletserver/tabletserver.go | 14 ++----- .../tabletserver/tabletserver_test.go | 40 +++++++++++++------ .../tabletserver/vstreamer/testenv/testenv.go | 7 ++-- 20 files changed, 129 insertions(+), 81 deletions(-) diff --git a/go/cmd/vtcombo/cli/main.go b/go/cmd/vtcombo/cli/main.go index a86b98cc250..16e625a6ae6 100644 --- a/go/cmd/vtcombo/cli/main.go +++ b/go/cmd/vtcombo/cli/main.go @@ -32,6 +32,7 @@ import ( "vitess.io/vitess/go/acl" "vitess.io/vitess/go/mysql/replication" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" @@ -84,6 +85,8 @@ In particular, it contains: tabletTypesToWait []topodatapb.TabletType env *vtenv.Environment + + srvTopoCounts *stats.CountersWithSingleLabel ) func init() { @@ -131,6 +134,7 @@ func init() { if err != nil { log.Fatalf("unable to initialize env: %v", err) } + srvTopoCounts = stats.NewCountersWithSingleLabel("ResilientSrvTopoServer", "Resilient srvtopo server operations", "type") } func startMysqld(uid uint32) (mysqld *mysqlctl.Mysqld, cnf *mysqlctl.Mycnf, err error) { @@ -234,7 +238,7 @@ func run(cmd *cobra.Command, args []string) (err error) { // to be the "internal" protocol that InitTabletMap registers. cmd.Flags().Set("tablet_manager_protocol", "internal") cmd.Flags().Set("tablet_protocol", "internal") - uid, err := vtcombo.InitTabletMap(env, ts, &tpb, mysqld, &dbconfigs.GlobalDBConfigs, schemaDir, startMysql) + uid, err := vtcombo.InitTabletMap(env, ts, &tpb, mysqld, &dbconfigs.GlobalDBConfigs, schemaDir, startMysql, srvTopoCounts) if err != nil { // ensure we start mysql in the event we fail here if startMysql { @@ -260,7 +264,7 @@ func run(cmd *cobra.Command, args []string) (err error) { } wr := wrangler.New(env, logutil.NewConsoleLogger(), ts, nil) - newUID, err := vtcombo.CreateKs(ctx, env, ts, &tpb, mysqld, &dbconfigs.GlobalDBConfigs, schemaDir, ks, true, uid, wr) + newUID, err := vtcombo.CreateKs(ctx, env, ts, &tpb, mysqld, &dbconfigs.GlobalDBConfigs, schemaDir, ks, true, uid, wr, srvTopoCounts) if err != nil { return err } @@ -297,7 +301,7 @@ func run(cmd *cobra.Command, args []string) (err error) { } // vtgate configuration and init - resilientServer = srvtopo.NewResilientServer(context.Background(), ts, "ResilientSrvTopoServer") + resilientServer = srvtopo.NewResilientServer(context.Background(), ts, srvTopoCounts) tabletTypes := make([]topodatapb.TabletType, 0, 1) if len(tabletTypesToWait) != 0 { diff --git a/go/cmd/vtexplain/cli/vtexplain.go b/go/cmd/vtexplain/cli/vtexplain.go index c8671cdb532..fe17d9af47c 100644 --- a/go/cmd/vtexplain/cli/vtexplain.go +++ b/go/cmd/vtexplain/cli/vtexplain.go @@ -22,6 +22,7 @@ import ( "os" "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/topo/memorytopo" @@ -186,7 +187,8 @@ func parseAndRun() error { } ctx := context.Background() ts := memorytopo.NewServer(ctx, vtexplain.Cell) - vte, err := vtexplain.Init(ctx, env, ts, vschema, schema, ksShardMap, opts) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + vte, err := vtexplain.Init(ctx, env, ts, vschema, schema, ksShardMap, opts, srvTopoCounts) if err != nil { return err } diff --git a/go/cmd/vtgate/cli/cli.go b/go/cmd/vtgate/cli/cli.go index c348c4d8296..312396a5a3c 100644 --- a/go/cmd/vtgate/cli/cli.go +++ b/go/cmd/vtgate/cli/cli.go @@ -25,6 +25,7 @@ import ( "vitess.io/vitess/go/acl" "vitess.io/vitess/go/exit" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/srvtopo" @@ -73,8 +74,14 @@ var ( PreRunE: servenv.CobraPreRunE, RunE: run, } + + srvTopoCounts *stats.CountersWithSingleLabel ) +func init() { + srvTopoCounts = stats.NewCountersWithSingleLabel("ResilientSrvTopoServer", "Resilient srvtopo server operations", "type") +} + // CheckCellFlags will check validation of cell and cells_to_watch flag // it will help to avoid strange behaviors when vtgate runs but actually does not work func CheckCellFlags(ctx context.Context, serv srvtopo.Server, cell string, cellsToWatch string) error { @@ -139,7 +146,7 @@ func run(cmd *cobra.Command, args []string) error { ts := topo.Open() defer ts.Close() - resilientServer = srvtopo.NewResilientServer(context.Background(), ts, "ResilientSrvTopoServer") + resilientServer = srvtopo.NewResilientServer(context.Background(), ts, srvTopoCounts) tabletTypes := make([]topodatapb.TabletType, 0, 1) for _, tt := range tabletTypesToWait { diff --git a/go/cmd/vttablet/cli/cli.go b/go/cmd/vttablet/cli/cli.go index 872d7a5ef69..ab212127457 100644 --- a/go/cmd/vttablet/cli/cli.go +++ b/go/cmd/vttablet/cli/cli.go @@ -27,6 +27,7 @@ import ( "vitess.io/vitess/go/acl" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/binlog" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" @@ -102,8 +103,14 @@ vttablet \ PreRunE: servenv.CobraPreRunE, RunE: run, } + + srvTopoCounts *stats.CountersWithSingleLabel ) +func init() { + srvTopoCounts = stats.NewCountersWithSingleLabel("TabletSrvTopo", "Resilient srvtopo server operations", "type") +} + func run(cmd *cobra.Command, args []string) error { servenv.Init() @@ -129,7 +136,7 @@ func run(cmd *cobra.Command, args []string) error { } ts := topo.Open() - qsc, err := createTabletServer(context.Background(), env, config, ts, tabletAlias) + qsc, err := createTabletServer(context.Background(), env, config, ts, tabletAlias, srvTopoCounts) if err != nil { ts.Close() return err @@ -249,7 +256,7 @@ func extractOnlineDDL() error { return nil } -func createTabletServer(ctx context.Context, env *vtenv.Environment, config *tabletenv.TabletConfig, ts *topo.Server, tabletAlias *topodatapb.TabletAlias) (*tabletserver.TabletServer, error) { +func createTabletServer(ctx context.Context, env *vtenv.Environment, config *tabletenv.TabletConfig, ts *topo.Server, tabletAlias *topodatapb.TabletAlias, srvTopoCounts *stats.CountersWithSingleLabel) (*tabletserver.TabletServer, error) { if tableACLConfig != "" { // To override default simpleacl, other ACL plugins must set themselves to be default ACL factory tableacl.Register("simpleacl", &simpleacl.Factory{}) @@ -258,7 +265,7 @@ func createTabletServer(ctx context.Context, env *vtenv.Environment, config *tab } // creates and registers the query service - qsc := tabletserver.NewTabletServer(ctx, env, "", config, ts, tabletAlias) + qsc := tabletserver.NewTabletServer(ctx, env, "", config, ts, tabletAlias, srvTopoCounts) servenv.OnRun(func() { qsc.Register() addStatusParts(qsc) diff --git a/go/vt/srvtopo/discover_test.go b/go/vt/srvtopo/discover_test.go index ca4774a1b84..81279b7d61a 100644 --- a/go/vt/srvtopo/discover_test.go +++ b/go/vt/srvtopo/discover_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/topo/memorytopo" querypb "vitess.io/vitess/go/vt/proto/query" @@ -59,7 +60,8 @@ func TestFindAllTargets(t *testing.T) { srvTopoCacheTTL = 1 * time.Second }() - rs := NewResilientServer(ctx, ts, "TestFindAllKeyspaceShards") + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + rs := NewResilientServer(ctx, ts, counts) // No keyspace / shards. ks, err := FindAllTargets(ctx, rs, "cell1", []topodatapb.TabletType{topodatapb.TabletType_PRIMARY}) diff --git a/go/vt/srvtopo/resilient_server.go b/go/vt/srvtopo/resilient_server.go index 0cbccbdb31c..78fc9134bce 100644 --- a/go/vt/srvtopo/resilient_server.go +++ b/go/vt/srvtopo/resilient_server.go @@ -70,7 +70,6 @@ const ( // - return the last known value of the data if there is an error type ResilientServer struct { topoServer *topo.Server - counts *stats.CountersWithSingleLabel *SrvKeyspaceWatcher *SrvVSchemaWatcher @@ -79,20 +78,13 @@ type ResilientServer struct { // NewResilientServer creates a new ResilientServer // based on the provided topo.Server. -func NewResilientServer(ctx context.Context, base *topo.Server, counterPrefix string) *ResilientServer { +func NewResilientServer(ctx context.Context, base *topo.Server, counts *stats.CountersWithSingleLabel) *ResilientServer { if srvTopoCacheRefresh > srvTopoCacheTTL { log.Fatalf("srv_topo_cache_refresh must be less than or equal to srv_topo_cache_ttl") } - metric := "" - if counterPrefix != "" { - metric = counterPrefix + "Counts" - } - counts := stats.NewCountersWithSingleLabel(metric, "Resilient srvtopo server operations", "type") - return &ResilientServer{ topoServer: base, - counts: counts, SrvKeyspaceWatcher: NewSrvKeyspaceWatcher(ctx, base, counts, srvTopoCacheRefresh, srvTopoCacheTTL), SrvVSchemaWatcher: NewSrvVSchemaWatcher(ctx, base, counts, srvTopoCacheRefresh, srvTopoCacheTTL), SrvKeyspaceNamesQuery: NewSrvKeyspaceNamesQuery(base, counts, srvTopoCacheRefresh, srvTopoCacheTTL), diff --git a/go/vt/srvtopo/resilient_server_test.go b/go/vt/srvtopo/resilient_server_test.go index fe248f56087..dbcf48ce176 100644 --- a/go/vt/srvtopo/resilient_server_test.go +++ b/go/vt/srvtopo/resilient_server_test.go @@ -28,6 +28,7 @@ import ( "github.com/google/safehtml/template" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/key" "github.com/stretchr/testify/assert" @@ -53,7 +54,8 @@ func TestGetSrvKeyspace(t *testing.T) { srvTopoCacheRefresh = 1 * time.Second }() - rs := NewResilientServer(ctx, ts, "TestGetSrvKeyspace") + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + rs := NewResilientServer(ctx, ts, counts) // Ask for a not-yet-created keyspace _, err := rs.GetSrvKeyspace(context.Background(), "test_cell", "test_ks") @@ -175,7 +177,7 @@ func TestGetSrvKeyspace(t *testing.T) { // Now simulate a topo service error and see that the last value is // cached for at least half of the expected ttl. errorTestStart := time.Now() - errorReqsBefore := rs.counts.Counts()[errorCategory] + errorReqsBefore := counts.Counts()[errorCategory] forceErr := topo.NewError(topo.Timeout, "test topo error") factory.SetError(forceErr) @@ -271,7 +273,7 @@ func TestGetSrvKeyspace(t *testing.T) { // Check that the expected number of errors were counted during the // interval - errorReqs := rs.counts.Counts()[errorCategory] + errorReqs := counts.Counts()[errorCategory] expectedErrors := int64(time.Since(errorTestStart) / srvTopoCacheRefresh) if errorReqs-errorReqsBefore > expectedErrors { t.Errorf("expected <= %v error requests got %d", expectedErrors, errorReqs-errorReqsBefore) @@ -370,7 +372,8 @@ func TestSrvKeyspaceCachedError(t *testing.T) { srvTopoCacheTTL = 1 * time.Second srvTopoCacheRefresh = 1 * time.Second }() - rs := NewResilientServer(ctx, ts, "TestSrvKeyspaceCachedErrors") + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + rs := NewResilientServer(ctx, ts, counts) // Ask for an unknown keyspace, should get an error. _, err := rs.GetSrvKeyspace(ctx, "test_cell", "unknown_ks") @@ -401,7 +404,8 @@ func TestGetSrvKeyspaceCreated(t *testing.T) { defer cancel() ts := memorytopo.NewServer(ctx, "test_cell") defer ts.Close() - rs := NewResilientServer(ctx, ts, "TestGetSrvKeyspaceCreated") + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + rs := NewResilientServer(ctx, ts, counts) // Set SrvKeyspace with value. want := &topodatapb.SrvKeyspace{} @@ -435,7 +439,8 @@ func TestWatchSrvVSchema(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "test_cell") - rs := NewResilientServer(ctx, ts, "TestWatchSrvVSchema") + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + rs := NewResilientServer(ctx, ts, counts) // mu protects watchValue and watchErr. mu := sync.Mutex{} @@ -529,7 +534,8 @@ func TestGetSrvKeyspaceNames(t *testing.T) { srvTopoCacheTTL = 1 * time.Second srvTopoCacheRefresh = 1 * time.Second }() - rs := NewResilientServer(ctx, ts, "TestGetSrvKeyspaceNames") + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + rs := NewResilientServer(ctx, ts, counts) // Set SrvKeyspace with value want := &topodatapb.SrvKeyspace{} @@ -614,7 +620,7 @@ func TestGetSrvKeyspaceNames(t *testing.T) { // Check that we only checked the topo service 1 or 2 times during the // period where we got the cached error. - cachedReqs, ok := rs.counts.Counts()[cachedCategory] + cachedReqs, ok := counts.Counts()[cachedCategory] if !ok || cachedReqs > 2 { t.Errorf("expected <= 2 cached requests got %v", cachedReqs) } @@ -640,7 +646,7 @@ func TestGetSrvKeyspaceNames(t *testing.T) { t.Errorf("GetSrvKeyspaceNames got %v want %v", names, wantNames) } - errorReqs, ok := rs.counts.Counts()[errorCategory] + errorReqs, ok := counts.Counts()[errorCategory] if !ok || errorReqs == 0 { t.Errorf("expected non-zero error requests got %v", errorReqs) } @@ -684,8 +690,8 @@ func TestSrvKeyspaceWatcher(t *testing.T) { srvTopoCacheTTL = 1 * time.Second srvTopoCacheRefresh = 1 * time.Second }() - - rs := NewResilientServer(ctx, ts, "TestGetSrvKeyspaceWatcher") + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + rs := NewResilientServer(ctx, ts, counts) var wmu sync.Mutex var wseen []watched @@ -811,7 +817,8 @@ func TestSrvKeyspaceListener(t *testing.T) { srvTopoCacheRefresh = 1 * time.Second }() - rs := NewResilientServer(ctx, ts, "TestGetSrvKeyspaceListener") + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + rs := NewResilientServer(ctx, ts, counts) cancelCtx, cancelFunc := context.WithCancel(context.Background()) var callbackCount atomic.Int32 diff --git a/go/vt/srvtopo/resolver_test.go b/go/vt/srvtopo/resolver_test.go index 95e6dbe620c..fae8bef1fb2 100644 --- a/go/vt/srvtopo/resolver_test.go +++ b/go/vt/srvtopo/resolver_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo/memorytopo" @@ -33,10 +34,11 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -func initResolver(t *testing.T, ctx context.Context, name string) *Resolver { +func initResolver(t *testing.T, ctx context.Context) *Resolver { cell := "cell1" ts := memorytopo.NewServer(ctx, cell) - rs := NewResilientServer(ctx, ts, name) + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + rs := NewResilientServer(ctx, ts, counts) // Create sharded keyspace and shards. if err := ts.CreateKeyspace(ctx, "sks", &topodatapb.Keyspace{}); err != nil { @@ -97,7 +99,7 @@ func initResolver(t *testing.T, ctx context.Context, name string) *Resolver { func TestResolveDestinations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - resolver := initResolver(t, ctx, "TestResolveDestinations") + resolver := initResolver(t, ctx) id1 := &querypb.Value{ Type: sqltypes.VarChar, diff --git a/go/vt/vtadmin/api.go b/go/vt/vtadmin/api.go index dce4ff041c9..1e83875de35 100644 --- a/go/vt/vtadmin/api.go +++ b/go/vt/vtadmin/api.go @@ -32,6 +32,7 @@ import ( "github.com/gorilla/mux" "github.com/patrickmn/go-cache" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/vtenv" "vitess.io/vitess/go/sets" @@ -2384,7 +2385,8 @@ func (api *API) VTExplain(ctx context.Context, req *vtadminpb.VTExplainRequest) } ts := memorytopo.NewServer(ctx, vtexplain.Cell) - vte, err := vtexplain.Init(ctx, api.env, ts, srvVSchema, schema, shardMap, &vtexplain.Options{ReplicationMode: "ROW"}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + vte, err := vtexplain.Init(ctx, api.env, ts, srvVSchema, schema, shardMap, &vtexplain.Options{ReplicationMode: "ROW"}, srvTopoCounts) if err != nil { return nil, fmt.Errorf("error initilaizing vtexplain: %w", err) } diff --git a/go/vt/vtcombo/tablet_map.go b/go/vt/vtcombo/tablet_map.go index 369a8138b5a..25cc28d366d 100644 --- a/go/vt/vtcombo/tablet_map.go +++ b/go/vt/vtcombo/tablet_map.go @@ -24,6 +24,7 @@ import ( "time" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/hook" @@ -84,6 +85,7 @@ func CreateTablet( tabletType topodatapb.TabletType, mysqld mysqlctl.MysqlDaemon, dbcfgs *dbconfigs.DBConfigs, + srvTopoCounts *stats.CountersWithSingleLabel, ) error { alias := &topodatapb.TabletAlias{ Cell: cell, @@ -91,7 +93,7 @@ func CreateTablet( } log.Infof("Creating %v tablet %v for %v/%v", tabletType, topoproto.TabletAliasString(alias), keyspace, shard) - controller := tabletserver.NewServer(ctx, env, topoproto.TabletAliasString(alias), ts, alias) + controller := tabletserver.NewServer(ctx, env, topoproto.TabletAliasString(alias), ts, alias, srvTopoCounts) initTabletType := tabletType if tabletType == topodatapb.TabletType_PRIMARY { initTabletType = topodatapb.TabletType_REPLICA @@ -173,6 +175,7 @@ func InitTabletMap( dbcfgs *dbconfigs.DBConfigs, schemaDir string, ensureDatabase bool, + srvTopoCounts *stats.CountersWithSingleLabel, ) (uint32, error) { tabletMap = make(map[uint32]*comboTablet) @@ -192,7 +195,7 @@ func InitTabletMap( var uid uint32 = 1 for _, kpb := range tpb.Keyspaces { var err error - uid, err = CreateKs(ctx, env, ts, tpb, mysqld, dbcfgs, schemaDir, kpb, ensureDatabase, uid, wr) + uid, err = CreateKs(ctx, env, ts, tpb, mysqld, dbcfgs, schemaDir, kpb, ensureDatabase, uid, wr, srvTopoCounts) if err != nil { return 0, err } @@ -293,6 +296,7 @@ func CreateKs( ensureDatabase bool, uid uint32, wr *wrangler.Wrangler, + srvTopoCounts *stats.CountersWithSingleLabel, ) (uint32, error) { keyspace := kpb.Name @@ -342,7 +346,7 @@ func CreateKs( replicas-- // create the primary - if err := CreateTablet(ctx, env, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_PRIMARY, mysqld, dbcfgs.Clone()); err != nil { + if err := CreateTablet(ctx, env, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_PRIMARY, mysqld, dbcfgs.Clone(), srvTopoCounts); err != nil { return 0, err } uid++ @@ -350,7 +354,7 @@ func CreateKs( for i := 0; i < replicas; i++ { // create a replica tablet - if err := CreateTablet(ctx, env, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_REPLICA, mysqld, dbcfgs.Clone()); err != nil { + if err := CreateTablet(ctx, env, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_REPLICA, mysqld, dbcfgs.Clone(), srvTopoCounts); err != nil { return 0, err } uid++ @@ -358,7 +362,7 @@ func CreateKs( for i := 0; i < rdonlys; i++ { // create a rdonly tablet - if err := CreateTablet(ctx, env, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_RDONLY, mysqld, dbcfgs.Clone()); err != nil { + if err := CreateTablet(ctx, env, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_RDONLY, mysqld, dbcfgs.Clone(), srvTopoCounts); err != nil { return 0, err } uid++ diff --git a/go/vt/vtexplain/vtexplain.go b/go/vt/vtexplain/vtexplain.go index e16dea89d2b..64f1ca3cea1 100644 --- a/go/vt/vtexplain/vtexplain.go +++ b/go/vt/vtexplain/vtexplain.go @@ -28,6 +28,7 @@ import ( "github.com/spf13/pflag" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vtenv" @@ -184,7 +185,7 @@ type TabletActions struct { } // Init sets up the fake execution environment -func Init(ctx context.Context, env *vtenv.Environment, ts *topo.Server, vSchemaStr, sqlSchema, ksShardMapStr string, opts *Options) (*VTExplain, error) { +func Init(ctx context.Context, env *vtenv.Environment, ts *topo.Server, vSchemaStr, sqlSchema, ksShardMapStr string, opts *Options, srvTopoCounts *stats.CountersWithSingleLabel) (*VTExplain, error) { // Verify options if opts.ReplicationMode != "ROW" && opts.ReplicationMode != "STATEMENT" { return nil, fmt.Errorf("invalid replication mode \"%s\"", opts.ReplicationMode) @@ -207,7 +208,7 @@ func Init(ctx context.Context, env *vtenv.Environment, ts *topo.Server, vSchemaS env: env, } vte.setGlobalTabletEnv(tabletEnv) - err = vte.initVtgateExecutor(ctx, ts, vSchemaStr, ksShardMapStr, opts) + err = vte.initVtgateExecutor(ctx, ts, vSchemaStr, ksShardMapStr, opts, srvTopoCounts) if err != nil { return nil, fmt.Errorf("initVtgateExecutor: %v", err.Error()) } diff --git a/go/vt/vtexplain/vtexplain_test.go b/go/vt/vtexplain/vtexplain_test.go index e7a6f4bdfc8..49bb94fedb1 100644 --- a/go/vt/vtexplain/vtexplain_test.go +++ b/go/vt/vtexplain/vtexplain_test.go @@ -28,6 +28,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vtenv" @@ -68,7 +69,8 @@ func initTest(ctx context.Context, ts *topo.Server, mode string, opts *Options, } opts.ExecutionMode = mode - vte, err := Init(ctx, vtenv.NewTestEnv(), ts, string(vSchema), string(schema), shardmap, opts) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + vte, err := Init(ctx, vtenv.NewTestEnv(), ts, string(vSchema), string(schema), shardmap, opts, srvTopoCounts) require.NoError(t, err, "vtexplain Init error\n%s", string(schema)) return vte } @@ -349,7 +351,8 @@ func TestInit(t *testing.T) { }` schema := "create table table_missing_primary_vindex (id int primary key)" ts := memorytopo.NewServer(ctx, Cell) - _, err := Init(ctx, vtenv.NewTestEnv(), ts, vschema, schema, "", defaultTestOpts()) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + _, err := Init(ctx, vtenv.NewTestEnv(), ts, vschema, schema, "", defaultTestOpts(), srvTopoCounts) require.Error(t, err) require.Contains(t, err.Error(), "missing primary col vindex") } diff --git a/go/vt/vtexplain/vtexplain_vtgate.go b/go/vt/vtexplain/vtexplain_vtgate.go index a5d9677f02a..22939efde20 100644 --- a/go/vt/vtexplain/vtexplain_vtgate.go +++ b/go/vt/vtexplain/vtexplain_vtgate.go @@ -28,6 +28,7 @@ import ( "vitess.io/vitess/go/cache/theine" "vitess.io/vitess/go/json2" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/streamlog" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/key" @@ -47,14 +48,14 @@ import ( vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" ) -func (vte *VTExplain) initVtgateExecutor(ctx context.Context, ts *topo.Server, vSchemaStr, ksShardMapStr string, opts *Options) error { +func (vte *VTExplain) initVtgateExecutor(ctx context.Context, ts *topo.Server, vSchemaStr, ksShardMapStr string, opts *Options, srvTopoCounts *stats.CountersWithSingleLabel) error { vte.explainTopo = &ExplainTopo{NumShards: opts.NumShards} vte.explainTopo.TopoServer = ts vte.healthCheck = discovery.NewFakeHealthCheck(nil) resolver := vte.newFakeResolver(ctx, opts, vte.explainTopo, Cell) - err := vte.buildTopology(ctx, ts, opts, vSchemaStr, ksShardMapStr, opts.NumShards) + err := vte.buildTopology(ctx, ts, opts, vSchemaStr, ksShardMapStr, opts.NumShards, srvTopoCounts) if err != nil { return err } @@ -92,7 +93,7 @@ func (vte *VTExplain) newFakeResolver(ctx context.Context, opts *Options, serv s return vtgate.NewResolver(srvResolver, serv, cell, sc) } -func (vte *VTExplain) buildTopology(ctx context.Context, ts *topo.Server, opts *Options, vschemaStr string, ksShardMapStr string, numShardsPerKeyspace int) error { +func (vte *VTExplain) buildTopology(ctx context.Context, ts *topo.Server, opts *Options, vschemaStr string, ksShardMapStr string, numShardsPerKeyspace int, srvTopoCounts *stats.CountersWithSingleLabel) error { vte.explainTopo.Lock.Lock() defer vte.explainTopo.Lock.Unlock() @@ -170,7 +171,7 @@ func (vte *VTExplain) buildTopology(ctx context.Context, ts *topo.Server, opts * log.Infof("registering test tablet %s for keyspace %s shard %s", hostname, ks, shard.Name) tablet := vte.healthCheck.AddFakeTablet(Cell, hostname, 1, ks, shard.Name, topodatapb.TabletType_PRIMARY, true, 1, nil, func(t *topodatapb.Tablet) queryservice.QueryService { - return vte.newTablet(ctx, vte.env, opts, t, ts) + return vte.newTablet(ctx, vte.env, opts, t, ts, srvTopoCounts) }) vte.explainTopo.TabletConns[hostname] = tablet.(*explainTablet) vte.explainTopo.KeyspaceShards[ks][shard.Name] = shard diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index b573fe29774..53e09445c17 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -23,6 +23,7 @@ import ( "strings" "sync" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/sidecardb" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vtenv" @@ -105,7 +106,7 @@ type explainTablet struct { var _ queryservice.QueryService = (*explainTablet)(nil) -func (vte *VTExplain) newTablet(ctx context.Context, env *vtenv.Environment, opts *Options, t *topodatapb.Tablet, ts *topo.Server) *explainTablet { +func (vte *VTExplain) newTablet(ctx context.Context, env *vtenv.Environment, opts *Options, t *topodatapb.Tablet, ts *topo.Server, srvTopoCounts *stats.CountersWithSingleLabel) *explainTablet { db := fakesqldb.New(nil) sidecardb.AddSchemaInitQueries(db, true, env.Parser()) @@ -120,7 +121,7 @@ func (vte *VTExplain) newTablet(ctx context.Context, env *vtenv.Environment, opt config.EnableTableGC = false // XXX much of this is cloned from the tabletserver tests - tsv := tabletserver.NewTabletServer(ctx, env, topoproto.TabletAliasString(t.Alias), config, ts, t.Alias) + tsv := tabletserver.NewTabletServer(ctx, env, topoproto.TabletAliasString(t.Alias), config, ts, t.Alias, srvTopoCounts) tablet := explainTablet{db: db, tsv: tsv, vte: vte, collationEnv: env.CollationEnv()} db.Handler = &tablet diff --git a/go/vt/vtexplain/vtexplain_vttablet_test.go b/go/vt/vtexplain/vtexplain_vttablet_test.go index a4350316d82..ddf24b90afa 100644 --- a/go/vt/vtexplain/vtexplain_vttablet_test.go +++ b/go/vt/vtexplain/vtexplain_vttablet_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vtenv" @@ -76,7 +77,8 @@ create table t2 ( ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, Cell) - vte, err := Init(ctx, vtenv.NewTestEnv(), ts, testVSchema, testSchema, "", opts) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + vte, err := Init(ctx, vtenv.NewTestEnv(), ts, testVSchema, testSchema, "", opts, srvTopoCounts) require.NoError(t, err) defer vte.Stop() @@ -141,14 +143,14 @@ create table test_partitioned ( tabletEnv, _ := newTabletEnvironment(ddls, defaultTestOpts(), env.CollationEnv()) vte.setGlobalTabletEnv(tabletEnv) - + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") tablet := vte.newTablet(ctx, env, defaultTestOpts(), &topodatapb.Tablet{ Keyspace: "ks_sharded", Shard: "-80", Alias: &topodatapb.TabletAlias{ Cell: Cell, }, - }, ts) + }, ts, srvTopoCounts) time.Sleep(10 * time.Millisecond) se := tablet.tsv.SchemaEngine() diff --git a/go/vt/vttablet/endtoend/framework/server.go b/go/vt/vttablet/endtoend/framework/server.go index 0258dee9186..42f2994a22c 100644 --- a/go/vt/vttablet/endtoend/framework/server.go +++ b/go/vt/vttablet/endtoend/framework/server.go @@ -23,6 +23,7 @@ import ( "net/http" "time" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/topo" @@ -78,7 +79,8 @@ func StartCustomServer(ctx context.Context, connParams, connAppDebugParams mysql } TopoServer = memorytopo.NewServer(ctx, "") - Server = tabletserver.NewTabletServer(ctx, vtenv.NewTestEnv(), "", cfg, TopoServer, &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + Server = tabletserver.NewTabletServer(ctx, vtenv.NewTestEnv(), "", cfg, TopoServer, &topodatapb.TabletAlias{}, srvTopoCounts) Server.Register() err := Server.StartService(Target, dbcfgs, nil /* mysqld */) if err != nil { diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index 3466a55133d..afc42113522 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -28,6 +28,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtenv" @@ -1528,7 +1529,8 @@ func newTestTabletServer(ctx context.Context, flags executorFlags, db *fakesqldb } dbconfigs := newDBConfigs(db) cfg.DB = dbconfigs - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) target := &querypb.Target{TabletType: topodatapb.TabletType_PRIMARY} err := tsv.StartService(target, dbconfigs, nil /* mysqld */) if cfg.TwoPCEnable { diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 6ecc46c68ab..dad637a6daf 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -28,7 +28,6 @@ import ( "sort" "strconv" "strings" - "sync" "sync/atomic" "syscall" "time" @@ -140,18 +139,13 @@ var _ queryservice.QueryService = (*TabletServer)(nil) var RegisterFunctions []func(Controller) // NewServer creates a new TabletServer based on the command line flags. -func NewServer(ctx context.Context, env *vtenv.Environment, name string, topoServer *topo.Server, alias *topodatapb.TabletAlias) *TabletServer { - return NewTabletServer(ctx, env, name, tabletenv.NewCurrentConfig(), topoServer, alias) +func NewServer(ctx context.Context, env *vtenv.Environment, name string, topoServer *topo.Server, alias *topodatapb.TabletAlias, srvTopoCounts *stats.CountersWithSingleLabel) *TabletServer { + return NewTabletServer(ctx, env, name, tabletenv.NewCurrentConfig(), topoServer, alias, srvTopoCounts) } -var ( - tsOnce sync.Once - srvTopoServer srvtopo.Server -) - // NewTabletServer creates an instance of TabletServer. Only the first // instance of TabletServer will expose its state variables. -func NewTabletServer(ctx context.Context, env *vtenv.Environment, name string, config *tabletenv.TabletConfig, topoServer *topo.Server, alias *topodatapb.TabletAlias) *TabletServer { +func NewTabletServer(ctx context.Context, env *vtenv.Environment, name string, config *tabletenv.TabletConfig, topoServer *topo.Server, alias *topodatapb.TabletAlias, srvTopoCounts *stats.CountersWithSingleLabel) *TabletServer { exporter := servenv.NewExporter(name, "Tablet") tsv := &TabletServer{ exporter: exporter, @@ -166,7 +160,7 @@ func NewTabletServer(ctx context.Context, env *vtenv.Environment, name string, c } tsv.QueryTimeout.Store(config.Oltp.QueryTimeout.Nanoseconds()) - tsOnce.Do(func() { srvTopoServer = srvtopo.NewResilientServer(ctx, topoServer, "TabletSrvTopo") }) + srvTopoServer := srvtopo.NewResilientServer(ctx, topoServer, srvTopoCounts) tabletTypeFunc := func() topodatapb.TabletType { if tsv.sm == nil || tsv.sm.Target() == nil { diff --git a/go/vt/vttablet/tabletserver/tabletserver_test.go b/go/vt/vttablet/tabletserver/tabletserver_test.go index 97777c0245f..9744b971946 100644 --- a/go/vt/vttablet/tabletserver/tabletserver_test.go +++ b/go/vt/vttablet/tabletserver/tabletserver_test.go @@ -32,6 +32,7 @@ import ( "vitess.io/vitess/go/mysql/config" "vitess.io/vitess/go/mysql/sqlerror" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/sidecardb" "vitess.io/vitess/go/vt/vtenv" @@ -1560,7 +1561,8 @@ func TestHandleExecUnknownError(t *testing.T) { defer cancel() logStats := tabletenv.NewLogStats(ctx, "TestHandleExecError") cfg := tabletenv.NewDefaultConfig() - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) defer tsv.handlePanicAndSendLogStats("select * from test_table", nil, logStats) panic("unknown exec error") } @@ -1730,7 +1732,8 @@ func TestHandleExecTabletError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cfg := tabletenv.NewDefaultConfig() - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError( @@ -1755,7 +1758,8 @@ func TestTerseErrors(t *testing.T) { cfg := tabletenv.NewDefaultConfig() cfg.TerseErrors = true cfg.SanitizeLogMessages = false - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() @@ -1789,7 +1793,8 @@ func TestSanitizeLogMessages(t *testing.T) { cfg := tabletenv.NewDefaultConfig() cfg.TerseErrors = false cfg.SanitizeLogMessages = true - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() @@ -1822,7 +1827,8 @@ func TestTerseErrorsNonSQLError(t *testing.T) { defer cancel() cfg := tabletenv.NewDefaultConfig() cfg.TerseErrors = true - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError( @@ -1847,7 +1853,8 @@ func TestSanitizeLogMessagesNonSQLError(t *testing.T) { cfg := tabletenv.NewDefaultConfig() cfg.TerseErrors = false cfg.SanitizeLogMessages = true - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError( @@ -1872,7 +1879,8 @@ func TestSanitizeMessagesBindVars(t *testing.T) { cfg := tabletenv.NewDefaultConfig() cfg.TerseErrors = true cfg.SanitizeLogMessages = true - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() @@ -1903,7 +1911,8 @@ func TestSanitizeMessagesNoBindVars(t *testing.T) { cfg := tabletenv.NewDefaultConfig() cfg.TerseErrors = true cfg.SanitizeLogMessages = true - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError(ctx, "", nil, vterrors.Errorf(vtrpcpb.Code_DEADLINE_EXCEEDED, "sensitive message"), nil) @@ -1921,7 +1930,8 @@ func TestTruncateErrorLen(t *testing.T) { defer cancel() cfg := tabletenv.NewDefaultConfig() cfg.TruncateErrorLen = 32 - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError( @@ -1952,7 +1962,8 @@ func TestTruncateMessages(t *testing.T) { TruncateErrLen: 52, }) require.NoError(t, err) - tsv := NewTabletServer(ctx, env, "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, env, "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() @@ -2006,7 +2017,8 @@ func TestTerseErrorsIgnoreFailoverInProgress(t *testing.T) { defer cancel() cfg := tabletenv.NewDefaultConfig() cfg.TerseErrors = true - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError(ctx, "select * from test_table where id = :a", @@ -2048,7 +2060,8 @@ func TestACLHUP(t *testing.T) { defer cancel() tableacl.Register("simpleacl", &simpleacl.Factory{}) cfg := tabletenv.NewDefaultConfig() - tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, vtenv.NewTestEnv(), "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) f, err := os.CreateTemp("", "tableacl") require.NoError(t, err) @@ -2564,7 +2577,8 @@ func setupTabletServerTest(t testing.TB, ctx context.Context, keyspaceName strin func setupTabletServerTestCustom(t testing.TB, ctx context.Context, cfg *tabletenv.TabletConfig, keyspaceName string, env *vtenv.Environment) (*fakesqldb.DB, *TabletServer) { db := setupFakeDB(t) sidecardb.AddSchemaInitQueries(db, true, env.Parser()) - tsv := NewTabletServer(ctx, env, "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + srvTopoCounts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + tsv := NewTabletServer(ctx, env, "TabletServerTest", cfg, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, srvTopoCounts) require.Equal(t, StateNotConnected, tsv.sm.State()) dbcfgs := newDBConfigs(db) target := &querypb.Target{ diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index 9c77ca18594..c09f73c0354 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -20,12 +20,12 @@ package testenv import ( "context" "fmt" - "math/rand" "os" "regexp" "strings" "vitess.io/vitess/go/json2" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/srvtopo" @@ -79,9 +79,8 @@ func Init(ctx context.Context) (*Env, error) { if err := te.TopoServ.CreateShard(ctx, te.KeyspaceName, te.ShardName); err != nil { panic(err) } - // Add a random suffix to metric name to avoid panic. Another option would have been to generate a random string. - suffix := rand.Int() - te.SrvTopo = srvtopo.NewResilientServer(ctx, te.TopoServ, "TestTopo"+fmt.Sprint(suffix)) + counts := stats.NewCountersWithSingleLabel("", "Resilient srvtopo server operations", "type") + te.SrvTopo = srvtopo.NewResilientServer(ctx, te.TopoServ, counts) cfg := vttest.Config{ Topology: &vttestpb.VTTestTopology{ From 6c6817748c99f43a3caeddf55c4a5036a3fbfb25 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Fri, 23 Feb 2024 14:52:06 +0100 Subject: [PATCH 13/16] discovery: Remove unused code (#15332) Signed-off-by: Dirkjan Bussink --- go/vt/discovery/utils.go | 22 ------ go/vt/discovery/utils_test.go | 134 ---------------------------------- 2 files changed, 156 deletions(-) delete mode 100644 go/vt/discovery/utils_test.go diff --git a/go/vt/discovery/utils.go b/go/vt/discovery/utils.go index 3a601830d35..253fead89a8 100644 --- a/go/vt/discovery/utils.go +++ b/go/vt/discovery/utils.go @@ -26,28 +26,6 @@ import ( // This file contains helper filter methods to process the unfiltered list of // tablets returned by HealthCheckImpl.GetTabletHealth*. -func TabletHealthReferenceListToValue(thl []*TabletHealth) []TabletHealth { - newTh := []TabletHealth{} - for _, th := range thl { - newTh = append(newTh, *th) - } - return newTh -} - -// RemoveUnhealthyTablets filters all unhealthy tablets out. -// NOTE: Non-serving tablets are considered healthy. -func RemoveUnhealthyTablets(tabletStatsList []TabletHealth) []TabletHealth { - result := make([]TabletHealth, 0, len(tabletStatsList)) - for _, ts := range tabletStatsList { - // Note we do not check the 'Serving' flag here. - if ts.LastError != nil || ts.Stats != nil && (ts.Stats.HealthError != "" || IsReplicationLagHigh(&ts)) { - continue - } - result = append(result, ts) - } - return result -} - func ParseTabletTypesAndOrder(tabletTypesStr string) ([]topodatapb.TabletType, bool, error) { inOrder := false if strings.HasPrefix(tabletTypesStr, InOrderHint) { diff --git a/go/vt/discovery/utils_test.go b/go/vt/discovery/utils_test.go deleted file mode 100644 index edca8c17602..00000000000 --- a/go/vt/discovery/utils_test.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright 2018 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package discovery - -import ( - "errors" - "testing" - - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -func TestRemoveUnhealthyTablets(t *testing.T) { - var testcases = []struct { - desc string - input []TabletHealth - want []TabletHealth - }{{ - desc: "tablets missing Stats", - input: []TabletHealth{replica(1), replica(2)}, - want: []TabletHealth{replica(1), replica(2)}, - }, { - desc: "all tablets healthy", - input: []TabletHealth{healthy(replica(1)), healthy(replica(2))}, - want: []TabletHealth{healthy(replica(1)), healthy(replica(2))}, - }, { - desc: "one unhealthy tablet (error)", - input: []TabletHealth{healthy(replica(1)), unhealthyError(replica(2))}, - want: []TabletHealth{healthy(replica(1))}, - }, { - desc: "one error tablet", - input: []TabletHealth{healthy(replica(1)), unhealthyLastError(replica(2))}, - want: []TabletHealth{healthy(replica(1))}, - }, { - desc: "one unhealthy tablet (lag)", - input: []TabletHealth{healthy(replica(1)), unhealthyLag(replica(2))}, - want: []TabletHealth{healthy(replica(1))}, - }, { - desc: "no filtering by tablet type", - input: []TabletHealth{healthy(primary(1)), healthy(replica(2)), healthy(rdonly(3))}, - want: []TabletHealth{healthy(primary(1)), healthy(replica(2)), healthy(rdonly(3))}, - }, { - desc: "non-serving tablets won't be removed", - input: []TabletHealth{notServing(healthy(replica(1)))}, - want: []TabletHealth{notServing(healthy(replica(1)))}, - }} - - for _, tc := range testcases { - t.Run(tc.desc, func(t *testing.T) { - got := RemoveUnhealthyTablets(tc.input) - if len(got) != len(tc.want) { - t.Errorf("test case '%v' failed: RemoveUnhealthyTablets(%v) = %#v, want: %#v", tc.desc, tc.input, got, tc.want) - } else { - for i := range tc.want { - if !got[i].DeepEqual(&tc.want[i]) { - t.Errorf("test case '%v' failed: RemoveUnhealthyTablets(%v) = %#v, want: %#v", tc.desc, tc.input, got, tc.want) - } - } - } - }) - } -} - -func primary(uid uint32) TabletHealth { - return minimalTabletStats(uid, topodatapb.TabletType_PRIMARY) -} - -func replica(uid uint32) TabletHealth { - return minimalTabletStats(uid, topodatapb.TabletType_REPLICA) -} - -func rdonly(uid uint32) TabletHealth { - return minimalTabletStats(uid, topodatapb.TabletType_RDONLY) -} - -func minimalTabletStats(uid uint32, tabletType topodatapb.TabletType) TabletHealth { - return TabletHealth{ - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Uid: uid}, - }, - Target: &querypb.Target{ - Keyspace: "test_keyspace", - Shard: "-80", - TabletType: tabletType, - }, - Serving: true, - } -} - -func healthy(ts TabletHealth) TabletHealth { - ts.Stats = &querypb.RealtimeStats{ - ReplicationLagSeconds: uint32(1), - } - return ts -} - -func unhealthyLag(ts TabletHealth) TabletHealth { - ts.Stats = &querypb.RealtimeStats{ - ReplicationLagSeconds: uint32(3600), - } - return ts -} - -func unhealthyError(ts TabletHealth) TabletHealth { - ts.Stats = &querypb.RealtimeStats{ - HealthError: "unhealthy", - } - return ts -} - -func unhealthyLastError(ts TabletHealth) TabletHealth { - ts.LastError = errors.New("err") - return ts -} - -func notServing(ts TabletHealth) TabletHealth { - ts.Serving = false - return ts -} From 4f8e465b3c437b8973569c91d0cd62503acbed5f Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Fri, 23 Feb 2024 19:11:26 +0100 Subject: [PATCH 14/16] wranger: Clean up leak check and use existing version (#15334) Signed-off-by: Dirkjan Bussink --- go/vt/wrangler/materializer_env_test.go | 76 +-------- go/vt/wrangler/materializer_test.go | 209 ++++++------------------ 2 files changed, 56 insertions(+), 229 deletions(-) diff --git a/go/vt/wrangler/materializer_env_test.go b/go/vt/wrangler/materializer_env_test.go index 3d5797a6bff..e3a3364c8fd 100644 --- a/go/vt/wrangler/materializer_env_test.go +++ b/go/vt/wrangler/materializer_env_test.go @@ -19,19 +19,14 @@ package wrangler import ( "context" "fmt" - "os" "regexp" - "runtime" "strconv" "strings" "sync" "testing" - "time" - - "go.uber.org/goleak" "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/tmutils" "vitess.io/vitess/go/vt/sqlparser" @@ -40,7 +35,6 @@ import ( "vitess.io/vitess/go/vt/vtenv" "vitess.io/vitess/go/vt/vttablet/tmclient" - _flag "vitess.io/vitess/go/internal/flag" querypb "vitess.io/vitess/go/vt/proto/query" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -61,66 +55,9 @@ type testMaterializerEnv struct { //---------------------------------------------- // testMaterializerEnv -// EnsureNoLeaks is a helper function to fail tests if there are goroutine leaks. -// At this moment we still have a lot of goroutine leaks in the unit tests in this package. -// So we only use this while debugging and fixing the leaks. Once fixed we will use this -// in TestMain instead of just logging the number of leaked goroutines. -func EnsureNoLeaks(t testing.TB) { - if t.Failed() { - return - } - err := ensureNoGoroutines() - if err != nil { - t.Fatal(err) - } -} - -func ensureNoGoroutines() error { - // These goroutines have been found to stay around. - // Need to investigate and fix the Vitess ones at some point, if we indeed find out that they are unintended leaks. - var leaksToIgnore = []goleak.Option{ - goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"), - goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), - goleak.IgnoreTopFunction("vitess.io/vitess/go/vt/dbconfigs.init.0.func1"), - goleak.IgnoreTopFunction("vitess.io/vitess/go/vt/vtgate.resetAggregators"), - goleak.IgnoreTopFunction("vitess.io/vitess/go/vt/vtgate.processQueryInfo"), - goleak.IgnoreTopFunction("github.com/patrickmn/go-cache.(*janitor).Run"), - } - - const ( - // give ample time for the goroutines to exit in CI. - waitTime = 100 * time.Millisecond - numIterations = 50 // 5 seconds - ) - var err error - for i := 0; i < numIterations; i++ { - err = goleak.Find(leaksToIgnore...) - if err == nil { - return nil - } - time.Sleep(waitTime) - } - return err -} - -func testMainWrapper(m *testing.M) int { - startingNumGoRoutines := runtime.NumGoroutine() - defer func() { - numGoroutines := runtime.NumGoroutine() - if numGoroutines > startingNumGoRoutines { - log.Infof("!!!!!!!!!!!! Wrangler unit tests Leaked %d goroutines", numGoroutines-startingNumGoRoutines) - } - }() - _flag.ParseFlagsForTest() - return m.Run() -} - -func TestMain(m *testing.M) { - os.Exit(testMainWrapper(m)) -} - -func newTestMaterializerEnv(t *testing.T, ctx context.Context, ms *vtctldatapb.MaterializeSettings, sources, targets []string) *testMaterializerEnv { +func newTestMaterializerEnv(t *testing.T, ms *vtctldatapb.MaterializeSettings, sources, targets []string) (*testMaterializerEnv, context.Context) { t.Helper() + ctx, cancel := context.WithCancel(context.Background()) env := &testMaterializerEnv{ ms: ms, sources: sources, @@ -167,7 +104,12 @@ func newTestMaterializerEnv(t *testing.T, ctx context.Context, ms *vtctldatapb.M if ms.Workflow != "" { env.expectValidation() } - return env + t.Cleanup(func() { + defer utils.EnsureNoLeaks(t) + env.close() + cancel() + }) + return env, ctx } func (env *testMaterializerEnv) expectValidation() { diff --git a/go/vt/wrangler/materializer_test.go b/go/vt/wrangler/materializer_test.go index bdeee1e6ac3..e7bb66f1003 100644 --- a/go/vt/wrangler/materializer_test.go +++ b/go/vt/wrangler/materializer_test.go @@ -65,10 +65,7 @@ func TestMoveTablesNoRoutingRules(t *testing.T) { SourceExpression: "select * from t1", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(100, mzCheckJournal, &sqltypes.Result{}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) @@ -93,11 +90,7 @@ func TestMigrateTables(t *testing.T) { SourceExpression: "select * from t1", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() - + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(100, mzCheckJournal, &sqltypes.Result{}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) env.tmc.expectVRQuery(200, insertPrefix, &sqltypes.Result{}) @@ -135,12 +128,8 @@ func TestMissingTables(t *testing.T) { SourceExpression: "select * from t3", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(100, mzCheckJournal, &sqltypes.Result{}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) env.tmc.expectVRQuery(200, insertPrefix, &sqltypes.Result{}) @@ -199,11 +188,7 @@ func TestMoveTablesAllAndExclude(t *testing.T) { } for _, tcase := range testCases { t.Run("", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(100, mzCheckJournal, &sqltypes.Result{}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) env.tmc.expectVRQuery(200, insertPrefix, &sqltypes.Result{}) @@ -231,11 +216,7 @@ func TestMoveTablesStopFlags(t *testing.T) { var err error t.Run("StopStartedAndStopAfterCopyFlags", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(100, mzCheckJournal, &sqltypes.Result{}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) // insert expects flag stop_after_copy to be true @@ -261,12 +242,7 @@ func TestMigrateVSchema(t *testing.T) { SourceExpression: "select * from t1", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() - + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(100, mzCheckJournal, &sqltypes.Result{}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) env.tmc.expectVRQuery(200, insertPrefix, &sqltypes.Result{}) @@ -294,11 +270,8 @@ func TestCreateLookupVindexFull(t *testing.T) { SourceKeyspace: "sourceks", TargetKeyspace: "targetks", } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) specs := &vschemapb.Keyspace{ Vindexes: map[string]*vschemapb.Vindex{ @@ -420,11 +393,9 @@ func TestCreateLookupVindexCreateDDL(t *testing.T) { SourceKeyspace: "sourceks", TargetKeyspace: "targetks", } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, _ := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) + vs := &vschemapb.Keyspace{ Sharded: true, Vindexes: map[string]*vschemapb.Vindex{ @@ -640,11 +611,8 @@ func TestCreateLookupVindexSourceVSchema(t *testing.T) { SourceKeyspace: "sourceks", TargetKeyspace: "targetks", } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, _ := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) specs := &vschemapb.Keyspace{ Vindexes: map[string]*vschemapb.Vindex{ @@ -879,11 +847,9 @@ func TestCreateLookupVindexTargetVSchema(t *testing.T) { SourceKeyspace: "sourceks", TargetKeyspace: "targetks", } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, _ := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) + sourcevs := &vschemapb.Keyspace{ Sharded: true, Vindexes: map[string]*vschemapb.Vindex{ @@ -1118,11 +1084,8 @@ func TestCreateLookupVindexSameKeyspace(t *testing.T) { SourceKeyspace: "ks", TargetKeyspace: "ks", } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, _ := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) specs := &vschemapb.Keyspace{ Vindexes: map[string]*vschemapb.Vindex{ @@ -1230,11 +1193,8 @@ func TestCreateCustomizedVindex(t *testing.T) { SourceKeyspace: "ks", TargetKeyspace: "ks", } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, _ := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) specs := &vschemapb.Keyspace{ Vindexes: map[string]*vschemapb.Vindex{ @@ -1343,11 +1303,8 @@ func TestCreateLookupVindexIgnoreNulls(t *testing.T) { SourceKeyspace: "ks", TargetKeyspace: "ks", } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, _ := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) specs := &vschemapb.Keyspace{ Vindexes: map[string]*vschemapb.Vindex{ @@ -1464,11 +1421,9 @@ func TestStopAfterCopyFlag(t *testing.T) { SourceKeyspace: "ks", TargetKeyspace: "ks", } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, _ := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) + specs := &vschemapb.Keyspace{ Vindexes: map[string]*vschemapb.Vindex{ "v": { @@ -1817,11 +1772,8 @@ func TestExternalizeVindex(t *testing.T) { SourceKeyspace: "sourceks", TargetKeyspace: "targetks", } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"-80", "80-"}) - defer env.close() + env, _ := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"-80", "80-"}) sourceVSchema := &vschemapb.Keyspace{ Sharded: true, @@ -1968,11 +1920,8 @@ func TestMaterializerOneToOne(t *testing.T) { topodatapb.TabletType_RDONLY, }), } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) env.tmc.expectVRQuery( @@ -2012,11 +1961,8 @@ func TestMaterializerManyToOne(t *testing.T) { CreateDdl: "t2ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"-80", "80-"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"-80", "80-"}, []string{"0"}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) env.tmc.expectVRQuery( @@ -2046,11 +1992,8 @@ func TestMaterializerOneToMany(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"-80", "80-"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"-80", "80-"}) vs := &vschemapb.Keyspace{ Sharded: true, @@ -2106,11 +2049,8 @@ func TestMaterializerManyToMany(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"-40", "40-"}, []string{"-80", "80-"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"-40", "40-"}, []string{"-80", "80-"}) vs := &vschemapb.Keyspace{ Sharded: true, @@ -2167,11 +2107,8 @@ func TestMaterializerMulticolumnVindex(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"-80", "80-"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"-80", "80-"}) vs := &vschemapb.Keyspace{ Sharded: true, @@ -2234,11 +2171,8 @@ func TestMaterializerDeploySchema(t *testing.T) { CreateDdl: "t2ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) delete(env.tmc.schema, "targetks.t2") @@ -2275,11 +2209,8 @@ func TestMaterializerCopySchema(t *testing.T) { CreateDdl: "t2ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) delete(env.tmc.schema, "targetks.t1") @@ -2313,11 +2244,8 @@ func TestMaterializerExplicitColumns(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"-80", "80-"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"-80", "80-"}) vs := &vschemapb.Keyspace{ Sharded: true, @@ -2376,11 +2304,8 @@ func TestMaterializerRenamedColumns(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"-80", "80-"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"-80", "80-"}) vs := &vschemapb.Keyspace{ Sharded: true, @@ -2444,11 +2369,8 @@ func TestMaterializerStopAfterCopy(t *testing.T) { CreateDdl: "t2ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) env.tmc.expectVRQuery(200, insertPrefix+`.*stop_after_copy:true`, &sqltypes.Result{}) @@ -2470,11 +2392,8 @@ func TestMaterializerNoTargetVSchema(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"-80", "80-"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"-80", "80-"}) vs := &vschemapb.Keyspace{ Sharded: true, @@ -2500,11 +2419,8 @@ func TestMaterializerNoDDL(t *testing.T) { CreateDdl: "", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) delete(env.tmc.schema, "targetks.t1") @@ -2544,7 +2460,6 @@ func TestMaterializerNoSourcePrimary(t *testing.T) { tmc: newTestMaterializerTMClient(), } env.wr = New(vtenv.NewTestEnv(), logutil.NewConsoleLogger(), env.topoServ, env.tmc) - defer env.close() tabletID := 100 for _, shard := range sources { @@ -2577,11 +2492,8 @@ func TestMaterializerTableMismatchNonCopy(t *testing.T) { CreateDdl: "", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) delete(env.tmc.schema, "targetks.t1") @@ -2601,11 +2513,8 @@ func TestMaterializerTableMismatchCopy(t *testing.T) { CreateDdl: "copy", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) delete(env.tmc.schema, "targetks.t1") @@ -2625,11 +2534,8 @@ func TestMaterializerNoSourceTable(t *testing.T) { CreateDdl: "copy", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) delete(env.tmc.schema, "targetks.t1") delete(env.tmc.schema, "sourceks.t1") @@ -2650,11 +2556,8 @@ func TestMaterializerSyntaxError(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) err := env.wr.Materialize(ctx, ms) @@ -2672,11 +2575,8 @@ func TestMaterializerNotASelect(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) err := env.wr.Materialize(ctx, ms) @@ -2694,11 +2594,8 @@ func TestMaterializerNoGoodVindex(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"-80", "80-"}) - defer env.close() + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"-80", "80-"}) vs := &vschemapb.Keyspace{ Sharded: true, @@ -2743,10 +2640,8 @@ func TestMaterializerComplexVindexExpression(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"-80", "80-"}) - defer env.close() + + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"-80", "80-"}) vs := &vschemapb.Keyspace{ Sharded: true, @@ -2786,10 +2681,8 @@ func TestMaterializerNoVindexInExpression(t *testing.T) { CreateDdl: "t1ddl", }}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"-80", "80-"}) - defer env.close() + + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"-80", "80-"}) vs := &vschemapb.Keyspace{ Sharded: true, @@ -3244,9 +3137,7 @@ func TestMaterializerSourceShardSelection(t *testing.T) { for _, tcase := range testcases { t.Run(tcase.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, tcase.sourceShards, tcase.targetShards) + env, ctx := newTestMaterializerEnv(t, ms, tcase.sourceShards, tcase.targetShards) if err := env.topoServ.SaveVSchema(ctx, "targetks", tcase.targetVSchema); err != nil { t.Fatal(err) } @@ -3255,7 +3146,7 @@ func TestMaterializerSourceShardSelection(t *testing.T) { t.Fatal(err) } } - defer env.close() + for i, targetShard := range tcase.targetShards { tabletID := 200 + i*10 env.tmc.expectVRQuery(tabletID, mzSelectFrozenQuery, &sqltypes.Result{}) @@ -3293,16 +3184,12 @@ func TestMoveTablesDDLFlag(t *testing.T) { SourceExpression: "select * from t1", }}, } - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() for onDDLAction := range binlogdatapb.OnDDLAction_value { t.Run(fmt.Sprintf("OnDDL Flag:%v", onDDLAction), func(t *testing.T) { - ctx, cancel := context.WithCancel(ctx) + env, ctx := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"}) + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() - env := newTestMaterializerEnv(t, ctx, ms, []string{"0"}, []string{"0"}) - defer env.close() - env.tmc.expectVRQuery(100, mzCheckJournal, &sqltypes.Result{}) env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{}) if onDDLAction == binlogdatapb.OnDDLAction_IGNORE.String() { @@ -3522,8 +3409,6 @@ func TestAddTablesToVSchema(t *testing.T) { // means that even if the target keyspace is sharded, the source // does not need to perform the in_keyrange filtering. func TestKeyRangesEqualOptimization(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() workflow := "testwf" sourceKs := "sourceks" targetKs := "targetks" @@ -3698,9 +3583,9 @@ func TestKeyRangesEqualOptimization(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - env := newTestMaterializerEnv(t, ctx, tc.ms, tc.sourceShards, tc.targetShards) - defer env.close() - + env, ctx := newTestMaterializerEnv(t, tc.ms, tc.sourceShards, tc.targetShards) + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() // Target is always sharded. err := env.wr.ts.SaveVSchema(ctx, targetKs, targetVSchema) require.NoError(t, err, "SaveVSchema failed: %v", err) From d8f771c695b5b82ec8b104b66587018b6e18f2fa Mon Sep 17 00:00:00 2001 From: Matt Layher Date: Fri, 23 Feb 2024 15:48:15 -0500 Subject: [PATCH 15/16] go/vt/discovery: use protobuf getters for SrvVschema (#15343) Signed-off-by: Matt Layher --- go/vt/discovery/keyspace_events.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/vt/discovery/keyspace_events.go b/go/vt/discovery/keyspace_events.go index c2567da9a87..014284ed5ee 100644 --- a/go/vt/discovery/keyspace_events.go +++ b/go/vt/discovery/keyspace_events.go @@ -391,8 +391,7 @@ func (kss *keyspaceState) getMoveTablesStatus(vs *vschemapb.SrvVSchema) (*MoveTa } // if there are no routing rules defined, then movetables is not in progress, exit early - if (vs.RoutingRules != nil && len(vs.RoutingRules.Rules) == 0) && - (vs.ShardRoutingRules != nil && len(vs.ShardRoutingRules.Rules) == 0) { + if len(vs.GetRoutingRules().GetRules()) == 0 && len(vs.GetShardRoutingRules().GetRules()) == 0 { return mtState, nil } From 47e137539bf70eade637edcbcbe151205ffebcee Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 25 Feb 2024 18:08:29 +0200 Subject: [PATCH 16/16] VReplication/OnlineDDL: reordering enum values (#15103) Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../vrepl_suite/testdata/enum-reorder/alter | 1 + .../testdata/enum-reorder/create.sql | 26 ++ go/vt/schemadiff/types.go | 4 +- go/vt/vttablet/onlineddl/executor.go | 1 - go/vt/vttablet/onlineddl/vrepl.go | 23 +- go/vt/vttablet/onlineddl/vrepl/columns.go | 2 +- .../vttablet/onlineddl/vrepl/columns_test.go | 245 ++++++++++++++++++ go/vt/vttablet/onlineddl/vrepl/foreign_key.go | 1 - .../vreplication/replicator_plan.go | 9 +- 9 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 go/test/endtoend/onlineddl/vrepl_suite/testdata/enum-reorder/alter create mode 100644 go/test/endtoend/onlineddl/vrepl_suite/testdata/enum-reorder/create.sql diff --git a/go/test/endtoend/onlineddl/vrepl_suite/testdata/enum-reorder/alter b/go/test/endtoend/onlineddl/vrepl_suite/testdata/enum-reorder/alter new file mode 100644 index 00000000000..6e011c14192 --- /dev/null +++ b/go/test/endtoend/onlineddl/vrepl_suite/testdata/enum-reorder/alter @@ -0,0 +1 @@ +change e e enum('blue', 'green', 'red') not null diff --git a/go/test/endtoend/onlineddl/vrepl_suite/testdata/enum-reorder/create.sql b/go/test/endtoend/onlineddl/vrepl_suite/testdata/enum-reorder/create.sql new file mode 100644 index 00000000000..84ebd4094c1 --- /dev/null +++ b/go/test/endtoend/onlineddl/vrepl_suite/testdata/enum-reorder/create.sql @@ -0,0 +1,26 @@ +drop table if exists onlineddl_test; +create table onlineddl_test ( + id int auto_increment, + i int not null, + e enum('red', 'green', 'blue') not null, + primary key(id) +) auto_increment=1; + +insert into onlineddl_test values (null, 11, 'red'); +insert into onlineddl_test values (null, 13, 'green'); +insert into onlineddl_test values (null, 17, 'blue'); + +drop event if exists onlineddl_test; +delimiter ;; +create event onlineddl_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into onlineddl_test values (null, 211, 'red'); + insert into onlineddl_test values (null, 213, 'green'); + insert into onlineddl_test values (null, 217, 'blue'); +end ;; diff --git a/go/vt/schemadiff/types.go b/go/vt/schemadiff/types.go index 30814dfc26c..a4edb09ec9b 100644 --- a/go/vt/schemadiff/types.go +++ b/go/vt/schemadiff/types.go @@ -120,8 +120,8 @@ const ( ) const ( - EnumReorderStrategyReject int = iota - EnumReorderStrategyAllow + EnumReorderStrategyAllow int = iota + EnumReorderStrategyReject ) // DiffHints is an assortment of rules for diffing entities diff --git a/go/vt/vttablet/onlineddl/executor.go b/go/vt/vttablet/onlineddl/executor.go index 30bb465a1f0..0fff5d920eb 100644 --- a/go/vt/vttablet/onlineddl/executor.go +++ b/go/vt/vttablet/onlineddl/executor.go @@ -2801,7 +2801,6 @@ func (e *Executor) evaluateDeclarativeDiff(ctx context.Context, onlineDDL *schem senv := schemadiff.NewEnv(e.env.Environment(), e.env.Environment().CollationEnv().DefaultConnectionCharset()) hints := &schemadiff.DiffHints{ AutoIncrementStrategy: schemadiff.AutoIncrementApplyHigher, - EnumReorderStrategy: schemadiff.EnumReorderStrategyAllow, } switch ddlStmt.(type) { case *sqlparser.CreateTable: diff --git a/go/vt/vttablet/onlineddl/vrepl.go b/go/vt/vttablet/onlineddl/vrepl.go index ef38fb7d012..847e40e3fbc 100644 --- a/go/vt/vttablet/onlineddl/vrepl.go +++ b/go/vt/vttablet/onlineddl/vrepl.go @@ -491,11 +491,26 @@ func (v *VRepl) analyzeTables(ctx context.Context, conn *dbconnpool.DBConnection for i := range v.sourceSharedColumns.Columns() { sourceColumn := v.sourceSharedColumns.Columns()[i] mappedColumn := v.targetSharedColumns.Columns()[i] - if sourceColumn.Type == vrepl.EnumColumnType && mappedColumn.Type != vrepl.EnumColumnType && mappedColumn.Charset != "" { - // A column is converted from ENUM type to textual type - v.targetSharedColumns.SetEnumToTextConversion(mappedColumn.Name, sourceColumn.EnumValues) - v.enumToTextMap[sourceColumn.Name] = sourceColumn.EnumValues + if sourceColumn.Type == vrepl.EnumColumnType { + switch { + // Either this is an ENUM column that stays an ENUM, or it is converted to a textual type. + // We take note of the enum values, and make it available in vreplication's Filter.Rule.ConvertEnumToText. + // This, in turn, will be used by vplayer (in TablePlan) like so: + // - In the binary log, enum values are integers. + // - Upon seeing this map, PlanBuilder will convert said int to the enum's logical string value. + // - And will apply the value as a string (`StringBindVariable`) in the query. + // What this allows is for enum values to have different ordering in the before/after table schema, + // so that for example you could modify an enum column: + // - from `('red', 'green', 'blue')` to `('red', 'blue')` + // - from `('red', 'green', 'blue')` to `('blue', 'red', 'green')` + case mappedColumn.Type == vrepl.EnumColumnType: + v.enumToTextMap[sourceColumn.Name] = sourceColumn.EnumValues + case mappedColumn.Charset != "": + v.enumToTextMap[sourceColumn.Name] = sourceColumn.EnumValues + v.targetSharedColumns.SetEnumToTextConversion(mappedColumn.Name, sourceColumn.EnumValues) + } } + if sourceColumn.IsIntegralType() && mappedColumn.Type == vrepl.EnumColumnType { v.intToEnumMap[sourceColumn.Name] = true } diff --git a/go/vt/vttablet/onlineddl/vrepl/columns.go b/go/vt/vttablet/onlineddl/vrepl/columns.go index 2937b1b2b2c..f2bb8f6d3f2 100644 --- a/go/vt/vttablet/onlineddl/vrepl/columns.go +++ b/go/vt/vttablet/onlineddl/vrepl/columns.go @@ -129,7 +129,7 @@ func isExpandedColumn(sourceColumn *Column, targetColumn *Column) (bool, string) return true, "source is unsigned, target is signed" } if sourceColumn.NumericPrecision <= targetColumn.NumericPrecision && !sourceColumn.IsUnsigned && targetColumn.IsUnsigned { - // e.g. INT SIGNED => INT UNSIGNED, INT SIGNED = BIGINT UNSIGNED + // e.g. INT SIGNED => INT UNSIGNED, INT SIGNED => BIGINT UNSIGNED return true, "target unsigned value exceeds source unsigned value" } if targetColumn.IsFloatingPoint() && !sourceColumn.IsFloatingPoint() { diff --git a/go/vt/vttablet/onlineddl/vrepl/columns_test.go b/go/vt/vttablet/onlineddl/vrepl/columns_test.go index b4d3ac9af58..201ffe55201 100644 --- a/go/vt/vttablet/onlineddl/vrepl/columns_test.go +++ b/go/vt/vttablet/onlineddl/vrepl/columns_test.go @@ -133,3 +133,248 @@ func TestGetSharedColumns(t *testing.T) { }) } } + +func TestGetExpandedColumnNames(t *testing.T) { + var ( + columnsA = &ColumnList{ + columns: []Column{ + { + Name: "c1", + IsNullable: true, + }, + { + Name: "c2", + IsNullable: true, + }, + { + Name: "c3", + IsNullable: false, + }, + }, + Ordinals: ColumnsMap{}, + } + columnsB = &ColumnList{ + columns: []Column{ + { + Name: "c1", + IsNullable: true, + }, + { + Name: "c2", + IsNullable: false, + }, + { + Name: "c3", + IsNullable: true, + }, + }, + Ordinals: ColumnsMap{}, + } + ) + tcases := []struct { + name string + sourceCol Column + targetCol Column + expanded bool + }{ + { + "both nullable", + Column{ + IsNullable: true, + }, + Column{ + IsNullable: true, + }, + false, + }, + { + "nullable to non nullable", + Column{ + IsNullable: true, + }, + Column{ + IsNullable: false, + }, + false, + }, + { + "non nullable to nullable", + Column{ + IsNullable: false, + }, + Column{ + IsNullable: true, + }, + true, + }, + { + "signed to unsigned", + Column{ + Type: IntegerColumnType, + NumericPrecision: 4, + IsUnsigned: false, + }, + Column{ + Type: IntegerColumnType, + NumericPrecision: 4, + IsUnsigned: true, + }, + true, + }, + { + "unsigned to signed", + Column{ + Type: IntegerColumnType, + NumericPrecision: 4, + IsUnsigned: true, + }, + Column{ + Type: IntegerColumnType, + NumericPrecision: 4, + IsUnsigned: false, + }, + true, + }, + { + "signed to smaller unsigned", + Column{ + Type: IntegerColumnType, + NumericPrecision: 8, + IsUnsigned: false, + }, + Column{ + Type: IntegerColumnType, + NumericPrecision: 4, + IsUnsigned: true, + }, + false, + }, + { + "same char length", + Column{ + CharacterMaximumLength: 20, + }, + Column{ + CharacterMaximumLength: 20, + }, + false, + }, + { + "reduced char length", + Column{ + CharacterMaximumLength: 20, + }, + Column{ + CharacterMaximumLength: 19, + }, + false, + }, + { + "increased char length", + Column{ + CharacterMaximumLength: 20, + }, + Column{ + CharacterMaximumLength: 21, + }, + true, + }, + { + "expand temporal", + Column{ + DataType: "time", + }, + Column{ + DataType: "timestamp", + }, + true, + }, + { + "expand temporal", + Column{ + DataType: "date", + }, + Column{ + DataType: "timestamp", + }, + true, + }, + { + "expand temporal", + Column{ + DataType: "date", + }, + Column{ + DataType: "datetime", + }, + true, + }, + { + "non expand temporal", + Column{ + DataType: "datetime", + }, + Column{ + DataType: "timestamp", + }, + false, + }, + { + "expand temporal", + Column{ + DataType: "timestamp", + }, + Column{ + DataType: "datetime", + }, + true, + }, + { + "expand enum", + Column{ + Type: EnumColumnType, + EnumValues: "'a', 'b'", + }, + Column{ + Type: EnumColumnType, + EnumValues: "'a', 'x'", + }, + true, + }, + { + "expand enum", + Column{ + Type: EnumColumnType, + EnumValues: "'a', 'b'", + }, + Column{ + Type: EnumColumnType, + EnumValues: "'a', 'b', 'c'", + }, + true, + }, + { + "reduce enum", + Column{ + Type: EnumColumnType, + EnumValues: "'a', 'b', 'c'", + }, + Column{ + Type: EnumColumnType, + EnumValues: "'a', 'b'", + }, + false, + }, + } + + expectedExpandedColumnNames := []string{"c3"} + expandedColumnNames, _ := GetExpandedColumnNames(columnsA, columnsB) + assert.Equal(t, expectedExpandedColumnNames, expandedColumnNames) + + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + expanded, _ := isExpandedColumn(&tcase.sourceCol, &tcase.targetCol) + assert.Equal(t, tcase.expanded, expanded) + }) + } +} diff --git a/go/vt/vttablet/onlineddl/vrepl/foreign_key.go b/go/vt/vttablet/onlineddl/vrepl/foreign_key.go index 8671badadc0..79e2df614f4 100644 --- a/go/vt/vttablet/onlineddl/vrepl/foreign_key.go +++ b/go/vt/vttablet/onlineddl/vrepl/foreign_key.go @@ -38,7 +38,6 @@ func RemovedForeignKeyNames( env := schemadiff.NewEnv(venv, venv.CollationEnv().DefaultConnectionCharset()) diffHints := schemadiff.DiffHints{ ConstraintNamesStrategy: schemadiff.ConstraintNamesIgnoreAll, - EnumReorderStrategy: schemadiff.EnumReorderStrategyAllow, } diff, err := schemadiff.DiffCreateTablesQueries(env, originalCreateTable, vreplCreateTable, &diffHints) if err != nil { diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index a328249d0e0..424daad4871 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -338,8 +338,13 @@ func (tp *TablePlan) bindFieldVal(field *querypb.Field, val *sqltypes.Value) (*q if enumValues, ok := tp.EnumValuesMap[field.Name]; ok && !val.IsNull() { // The fact that this field has a EnumValuesMap entry, means we must // use the enum's text value as opposed to the enum's numerical value. - // Once known use case is with Online DDL, when a column is converted from - // ENUM to a VARCHAR/TEXT. + // This may be needed in Online DDL, when the enum column could be modified: + // - Either from ENUM to a text type (VARCHAR/TEXT) + // - Or from ENUM to another ENUM with different value ordering, + // e.g. from `('red', 'green', 'blue')` to `('red', 'blue')`. + // By applying the textual value of an enum we eliminate the ordering concern. + // In non-Online DDL this shouldn't be a concern because the schema is static, + // and so passing the enum's numerical value is sufficient. enumValue, enumValueOK := enumValues[val.ToString()] if !enumValueOK { return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "Invalid enum value: %v for field %s", val, field.Name)