Merge remote-tracking branch 'upstream/main' into reference-dml-fix

Signed-off-by: Harshit Gangal <[email protected]>

changelog/22.0/22.0.0/summary.md

@@ -10,7 +10,7 @@
   - **[VTOrc Config File Changes](#vtorc-config-file-changes)**
 - **[Minor Changes](#minor-changes)**
   - **[VTTablet Flags](#flags-vttablet)**
-
+  - **[Topology read concurrency behaviour changes](#topo-read-concurrency-changes)**
 
 ## <a id="major-changes"/>Major Changes</a>
 
@@ -67,3 +67,9 @@ To upgrade to the newer version of the configuration file, first switch to using
 - `twopc_abandon_age` flag now supports values in the time.Duration format (e.g., 1s, 2m, 1h). 
 While the flag will continue to accept float values (interpreted as seconds) for backward compatibility, 
 **float inputs are deprecated** and will be removed in a future release.
+
+### <a id="topo-read-concurrency-changes"/>`--topo_read_concurrency` behaviour changes
+
+The `--topo_read_concurrency` flag was added to all components that access the topology and the provided limit is now applied separately for each global or local cell _(default `32`)_.
+
+All topology read calls _(`Get`, `GetVersion`, `List` and `ListDir`)_ now respect this per-cell limit. Previous to this version a single limit was applied to all cell calls and it was not respected by many topology calls.

config/mycnf/default.cnf

@@ -16,10 +16,6 @@ secure-file-priv = {{.SecureFilePriv}}
 
 server-id = {{.ServerID}}
 
-# all db instances should skip starting replication threads - that way we can do any
-# additional configuration (like enabling semi-sync) before we connect to
-# the source.
-skip_slave_start
 socket = {{.SocketFile}}
 tmpdir = {{.TmpDir}}
 

config/mycnf/mariadb10.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MariaDB 10 is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_slave_start
+
 # Semi-sync replication is required for automated unplanned failover
 # (when the primary goes away). Here we just load the plugin so it's
 # available if desired, but it's disabled at startup.

config/mycnf/mysql57.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MySQL 5.7 is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_slave_start
+
 # MySQL 5.7 does not enable the binary log by default, and 
 # info repositories default to file
 

config/mycnf/mysql80.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MySQL 8.0 is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_slave_start
+
 # MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories
 # It does not enable GTIDs or enforced GTID consistency
 

config/mycnf/mysql8026.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MySQL 8.0.26 or later is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_replica_start
+
 # MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories
 # It does not enable GTIDs or enforced GTID consistency
 

config/mycnf/mysql84.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MySQL 8.4.0 or later is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_replica_start
+
 # MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories
 # It does not enable GTIDs or enforced GTID consistency
 

go/flags/endtoend/vtbackup.txt

@@ -231,6 +231,7 @@ Flags:
       --topo_global_root string                                     the path of the global topology data in the global topology server
       --topo_global_server_address string                           the address of the global topology server
       --topo_implementation string                                  the topology implementation to use
+      --topo_read_concurrency int                                   Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo_zk_auth_file string                                    auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo_zk_base_timeout duration                               zk base timeout (see zk.Connect) (default 30s)
       --topo_zk_max_concurrency int                                 maximum number of pending requests to send to a Zookeeper server. (default 64)

go/flags/endtoend/vtcombo.txt

@@ -22,15 +22,7 @@ Flags:
       --backup_storage_number_blocks int                                 if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2)
       --bind-address string                                              Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system.
       --binlog-in-memory-decompressor-max-size uint                      This value sets the uncompressed transaction payload size at which we switch from in-memory buffer based decompression to the slower streaming mode. (default 134217728)
-      --binlog_host string                                               PITR restore parameter: hostname/IP of binlog server.
-      --binlog_password string                                           PITR restore parameter: password of binlog server.
       --binlog_player_protocol string                                    the protocol to download binlogs from a vttablet (default "grpc")
-      --binlog_port int                                                  PITR restore parameter: port of binlog server.
-      --binlog_ssl_ca string                                             PITR restore parameter: Filename containing TLS CA certificate to verify binlog server TLS certificate against.
-      --binlog_ssl_cert string                                           PITR restore parameter: Filename containing mTLS client certificate to present to binlog server as authentication.
-      --binlog_ssl_key string                                            PITR restore parameter: Filename containing mTLS client private key for use in binlog server authentication.
-      --binlog_ssl_server_name string                                    PITR restore parameter: TLS server name (common name) to verify against for the binlog server we are connecting to (If not set: use the hostname or IP supplied in --binlog_host).
-      --binlog_user string                                               PITR restore parameter: username of binlog server.
       --buffer_drain_concurrency int                                     Maximum number of requests retried simultaneously. More concurrency will increase the load on the PRIMARY vttablet when draining the buffer. (default 1)
       --buffer_keyspace_shards string                                    If not empty, limit buffering to these entries (comma separated). Entry format: keyspace or keyspace/shard. Requires --enable_buffer=true.
       --buffer_max_failover_duration duration                            Stop buffering completely if a failover takes longer than this duration. (default 20s)
@@ -261,7 +253,6 @@ Flags:
       --onclose_timeout duration                                         wait no more than this for OnClose handlers before stopping (default 10s)
       --onterm_timeout duration                                          wait no more than this for OnTermSync handlers before stopping (default 10s)
       --pid_file string                                                  If set, the process will write its pid to the named file, and delete it on graceful shutdown.
-      --pitr_gtid_lookup_timeout duration                                PITR restore parameter: timeout for fetching gtid from timestamp. (default 1m0s)
       --planner-version string                                           Sets the default planner to use when the session has not changed it. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right
       --pool_hostname_resolve_interval duration                          if set force an update to all hostnames and reconnect if changed, defaults to 0 (disabled)
       --port int                                                         port for the server
@@ -374,7 +365,7 @@ Flags:
       --topo_global_root string                                          the path of the global topology data in the global topology server
       --topo_global_server_address string                                the address of the global topology server
       --topo_implementation string                                       the topology implementation to use
-      --topo_read_concurrency int                                        Concurrency of topo reads. (default 32)
+      --topo_read_concurrency int                                        Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo_zk_auth_file string                                         auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo_zk_base_timeout duration                                    zk base timeout (see zk.Connect) (default 30s)
       --topo_zk_max_concurrency int                                      maximum number of pending requests to send to a Zookeeper server. (default 64)

go/flags/endtoend/vtctld.txt

@@ -164,7 +164,7 @@ Flags:
       --topo_global_root string                                          the path of the global topology data in the global topology server
       --topo_global_server_address string                                the address of the global topology server
       --topo_implementation string                                       the topology implementation to use
-      --topo_read_concurrency int                                        Concurrency of topo reads. (default 32)
+      --topo_read_concurrency int                                        Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo_zk_auth_file string                                         auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo_zk_base_timeout duration                                    zk base timeout (see zk.Connect) (default 30s)
       --topo_zk_max_concurrency int                                      maximum number of pending requests to send to a Zookeeper server. (default 64)

go/flags/endtoend/vtgate.txt

@@ -223,7 +223,7 @@ Flags:
       --topo_global_root string                                          the path of the global topology data in the global topology server
       --topo_global_server_address string                                the address of the global topology server
       --topo_implementation string                                       the topology implementation to use
-      --topo_read_concurrency int                                        Concurrency of topo reads. (default 32)
+      --topo_read_concurrency int                                        Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo_zk_auth_file string                                         auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo_zk_base_timeout duration                                    zk base timeout (see zk.Connect) (default 30s)
       --topo_zk_max_concurrency int                                      maximum number of pending requests to send to a Zookeeper server. (default 64)

go/flags/endtoend/vtorc.txt

@@ -98,7 +98,7 @@ Flags:
       --topo_global_root string                                     the path of the global topology data in the global topology server
       --topo_global_server_address string                           the address of the global topology server
       --topo_implementation string                                  the topology implementation to use
-      --topo_read_concurrency int                                   Concurrency of topo reads. (default 32)
+      --topo_read_concurrency int                                   Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo_zk_auth_file string                                    auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo_zk_base_timeout duration                               zk base timeout (see zk.Connect) (default 30s)
       --topo_zk_max_concurrency int                                 maximum number of pending requests to send to a Zookeeper server. (default 64)

go/flags/endtoend/vttablet.txt

@@ -57,20 +57,12 @@ Flags:
       --backup_storage_number_blocks int                                 if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2)
       --bind-address string                                              Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system.
       --binlog-in-memory-decompressor-max-size uint                      This value sets the uncompressed transaction payload size at which we switch from in-memory buffer based decompression to the slower streaming mode. (default 134217728)
-      --binlog_host string                                               PITR restore parameter: hostname/IP of binlog server.
-      --binlog_password string                                           PITR restore parameter: password of binlog server.
       --binlog_player_grpc_ca string                                     the server ca to use to validate servers when connecting
       --binlog_player_grpc_cert string                                   the cert to use to connect
       --binlog_player_grpc_crl string                                    the server crl to use to validate server certificates when connecting
       --binlog_player_grpc_key string                                    the key to use to connect
       --binlog_player_grpc_server_name string                            the server name to use to validate server certificate
       --binlog_player_protocol string                                    the protocol to download binlogs from a vttablet (default "grpc")
-      --binlog_port int                                                  PITR restore parameter: port of binlog server.
-      --binlog_ssl_ca string                                             PITR restore parameter: Filename containing TLS CA certificate to verify binlog server TLS certificate against.
-      --binlog_ssl_cert string                                           PITR restore parameter: Filename containing mTLS client certificate to present to binlog server as authentication.
-      --binlog_ssl_key string                                            PITR restore parameter: Filename containing mTLS client private key for use in binlog server authentication.
-      --binlog_ssl_server_name string                                    PITR restore parameter: TLS server name (common name) to verify against for the binlog server we are connecting to (If not set: use the hostname or IP supplied in --binlog_host).
-      --binlog_user string                                               PITR restore parameter: username of binlog server.
       --builtinbackup-file-read-buffer-size uint                         read files using an IO buffer of this many bytes. Golang defaults are used when set to 0.
       --builtinbackup-file-write-buffer-size uint                        write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152)
       --builtinbackup-incremental-restore-path string                    the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed.
@@ -259,7 +251,6 @@ Flags:
       --onterm_timeout duration                                          wait no more than this for OnTermSync handlers before stopping (default 10s)
       --opentsdb_uri string                                              URI of opentsdb /api/put method
       --pid_file string                                                  If set, the process will write its pid to the named file, and delete it on graceful shutdown.
-      --pitr_gtid_lookup_timeout duration                                PITR restore parameter: timeout for fetching gtid from timestamp. (default 1m0s)
       --pool_hostname_resolve_interval duration                          if set force an update to all hostnames and reconnect if changed, defaults to 0 (disabled)
       --port int                                                         port for the server
       --pprof strings                                                    enable profiling
@@ -376,6 +367,7 @@ Flags:
       --topo_global_root string                                          the path of the global topology data in the global topology server
       --topo_global_server_address string                                the address of the global topology server
       --topo_implementation string                                       the topology implementation to use
+      --topo_read_concurrency int                                        Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo_zk_auth_file string                                         auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo_zk_base_timeout duration                                    zk base timeout (see zk.Connect) (default 30s)
       --topo_zk_max_concurrency int                                      maximum number of pending requests to send to a Zookeeper server. (default 64)

go/mysql/flavor_filepos.go

@@ -84,7 +84,7 @@ func (flv *filePosFlavor) gtidMode(c *Conn) (string, error) {
 
 // serverUUID is part of the Flavor interface.
 func (flv *filePosFlavor) serverUUID(c *Conn) (string, error) {
-	// keep @@global as lowercase, as some servers like the Ripple binlog server only honors a lowercase `global` value
+	// keep @@global as lowercase, as some servers like a binlog server only honors a lowercase `global` value
 	qr, err := c.ExecuteFetch("SELECT @@global.server_uuid", 1, false)
 	if err != nil {
 		return "", err