vitessio · mattlord · Dec 8, 2024 · Dec 9, 2024 · Dec 9, 2024 · Dec 9, 2024
@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard schemadiff_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_basic | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_cellalias | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_copy_parallel | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_foreign_key_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_mariadb_to_mysql | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_multi_tenant | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_partial_movetables_and_materialize | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_v2 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_vtctldclient_vdiff2_movetables_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -111,7 +111,7 @@ jobs:
 
     - name: Upload coverage reports to codecov.io
       if: steps.changes.outputs.changed_files == 'true'
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # https://github.com/codecov/codecov-action/releases/tag/v5.0.7
       with:
         fail_ci_if_error: true
         verbose: true

@@ -163,6 +163,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql57"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql84"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -163,6 +163,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql57"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql84"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

@@ -8,6 +8,7 @@
   - **[RPC Changes](#rpc-changes)**
   - **[Prefer not promoting a replica that is currently taking a backup](#reparents-prefer-not-backing-up)**
   - **[VTOrc Config File Changes](#vtorc-config-file-changes)**
+  - **[Support for More Efficient JSON Replication](#efficient-json-replication)**
 - **[Minor Changes](#minor-changes)**
   - **[VTTablet Flags](#flags-vttablet)**
   - **[Topology read concurrency behaviour changes](#topo-read-concurrency-changes)**
@@ -59,6 +60,12 @@ The following fields can be dynamically changed -
 
 To upgrade to the newer version of the configuration file, first switch to using the flags in your current deployment before upgrading. Then you can switch to using the configuration file in the newer release.
 
+### <a id="efficient-json-replication"/>Support for More Efficient JSON Replication</a>
+
+In [#7345](https://github.com/vitessio/vitess/pull/17345) we added support for [`--binlog-row-value-options=PARTIAL_JSON`](https://dev.mysql.com/doc/refman/en/replication-options-binary-log.html#sysvar_binlog_row_value_options). You can read more about [this feature added to MySQL 8.0 here](https://dev.mysql.com/blog-archive/efficient-json-replication-in-mysql-8-0/).
+
+If you are using MySQL 8.0 or later and using JSON columns, you can now enable this MySQL feature across your Vitess cluster(s) to lower the disk space needed for binary logs and improve the CPU and memory usage in both `mysqld` (standard intrashard MySQL replication) and `vttablet` ([VReplication](https://vitess.io/docs/reference/vreplication/vreplication/)) without losing any capabilities or features.
+
 
 ## <a id="minor-changes"/>Minor Changes</a>
 

@@ -17,6 +17,7 @@ limitations under the License.
 package binlog
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"math"
@@ -25,9 +26,12 @@ import (
 	"vitess.io/vitess/go/hack"
 	"vitess.io/vitess/go/mysql/format"
 	"vitess.io/vitess/go/mysql/json"
+	"vitess.io/vitess/go/sqltypes"
+	"vitess.io/vitess/go/vt/sqlparser"
+	"vitess.io/vitess/go/vt/vterrors"
+
 	querypb "vitess.io/vitess/go/vt/proto/query"
 	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
-	"vitess.io/vitess/go/vt/vterrors"
 )
 
 /*
@@ -44,6 +48,14 @@ https://github.com/shyiko/mysql-binlog-connector-java/pull/119/files
 https://github.com/noplay/python-mysql-replication/blob/175df28cc8b536a68522ff9b09dc5440adad6094/pymysqlreplication/packet.py
 */
 
+type jsonDiffOp uint8
+
+const (
+	jsonDiffOpReplace = jsonDiffOp(iota)
+	jsonDiffOpInsert
+	jsonDiffOpRemove
+)
+
 // ParseBinaryJSON provides the parsing function from the mysql binary json
 // representation to a JSON value instance.
 func ParseBinaryJSON(data []byte) (*json.Value, error) {
@@ -60,6 +72,85 @@ func ParseBinaryJSON(data []byte) (*json.Value, error) {
 	return node, nil
 }
 
+// ParseBinaryJSONDiff provides the parsing function from the binary MySQL
+// JSON diff representation to an SQL expression.
+func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) {
+	if len(data) == 0 {
+		return sqltypes.MakeTrusted(sqltypes.Expression, data), nil
+	}
+
+	diff := bytes.Buffer{}
+	// Reasonable estimate of the space we'll need to build the SQL
+	// expression in order to try and avoid reallocations w/o
+	// overallocating too much.
+	diff.Grow(len(data) + 80)
+	pos := 0
+	outer := false
+	innerStr := ""
+
+	for pos < len(data) {
+		opType := jsonDiffOp(data[pos])
+		pos++
+		if outer {
+			innerStr = diff.String()
+			diff.Reset()
+		}
+		switch opType {
+		case jsonDiffOpReplace:
+			diff.WriteString("JSON_REPLACE(")
+		case jsonDiffOpInsert:
+			diff.WriteString("JSON_INSERT(")
+		case jsonDiffOpRemove:
+			diff.WriteString("JSON_REMOVE(")
+		default:
+			// Can be a JSON null.
+			js, err := ParseBinaryJSON(data)
+			if err == nil && js.Type() == json.TypeNull {
+				return sqltypes.MakeTrusted(sqltypes.Expression, js.MarshalSQLTo(nil)), nil
+			}
+			return sqltypes.Value{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT,
+				"invalid JSON diff operation: %d", opType)
+		}
+		if outer {
+			diff.WriteString(innerStr)
+			diff.WriteString(", ")
+		} else { // Only the inner most function has the field name
+			diff.WriteString("%s, ") // This will later be replaced by the field name
+		}
+		outer = true
+
+		pathLen, readTo := readVariableLength(data, pos)
+		pos = readTo
+		path := data[pos : pos+pathLen]
+		pos += pathLen
+		// We have to specify the unicode character set for the strings we
+		// use in the expression as the connection can be using a different
+		// character set (e.g. vreplication always uses set names binary).
+		diff.WriteString(sqlparser.Utf8mb4Str)
+		diff.WriteByte('\'')
+		diff.Write(path)
+		diff.WriteByte('\'')
+		if opType == jsonDiffOpRemove { // No value for remove
+			diff.WriteByte(')')
+			continue
+		}
+
+		diff.WriteString(", ")
+		valueLen, readTo := readVariableLength(data, pos)
+		pos = readTo
+		value, err := ParseBinaryJSON(data[pos : pos+valueLen])
+		if err != nil {
+			return sqltypes.Value{}, vterrors.Wrapf(err,
+				"cannot read JSON diff value for path %q", path)
+		}
+		pos += valueLen
+		diff.Write(value.MarshalSQLTo(nil))
+		diff.WriteByte(')')
+	}
+
+	return sqltypes.MakeTrusted(sqltypes.Expression, diff.Bytes()), nil
+}
+
 // jsonDataType has the values used in the mysql json binary representation to denote types.
 // We have string, literal(true/false/null), number, object or array types.
 // large object => doc size > 64K: you get pointers instead of inline values.
@@ -315,7 +406,7 @@ func binparserOpaque(_ jsonDataType, data []byte, pos int) (node *json.Value, er
 		precision := decimalData[0]
 		scale := decimalData[1]
 		metadata := (uint16(precision) << 8) + uint16(scale)
-		val, _, err := CellValue(decimalData, 2, TypeNewDecimal, metadata, &querypb.Field{Type: querypb.Type_DECIMAL})
+		val, _, err := CellValue(decimalData, 2, TypeNewDecimal, metadata, &querypb.Field{Type: querypb.Type_DECIMAL}, false)
 		if err != nil {
 			return nil, err
 		}