feat: add benchmarking to compare MySQL and vitess performance on the…

… fuzzer generated queries

Signed-off-by: Manan Gupta <[email protected]>

go/test/endtoend/utils/utils.go

@@ -304,7 +304,7 @@ func WaitForTableDeletions(ctx context.Context, t *testing.T, vtgateProcess clus
 }
 
 // WaitForColumn waits for a table's column to be present
-func WaitForColumn(t *testing.T, vtgateProcess cluster.VtgateProcess, ks, tbl, col string) error {
+func WaitForColumn(t testing.TB, vtgateProcess cluster.VtgateProcess, ks, tbl, col string) error {
 	timeout := time.After(60 * time.Second)
 	for {
 		select {

go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package foreignkey
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"math/rand"
@@ -28,6 +29,7 @@ import (
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/stretchr/testify/require"
 
+	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/sqltypes"
 	"vitess.io/vitess/go/test/endtoend/utils"
 	"vitess.io/vitess/go/vt/log"
@@ -671,14 +673,55 @@ func TestFkFuzzTest(t *testing.T) {
 	}
 }
 
-func validateReplication(t *testing.T) {
-	for _, keyspace := range clusterInstance.Keyspaces {
-		for _, shard := range keyspace.Shards {
-			for _, vttablet := range shard.Vttablets {
-				if vttablet.Type != "primary" {
-					checkReplicationHealthy(t, vttablet)
-				}
-			}
+// BenchmarkFkFuzz benchmarks the performance of Vitess unmanaged, Vitess managed and vanilla MySQL performance on a given set of queries generated by the fuzzer.
+func BenchmarkFkFuzz(b *testing.B) {
+	maxValForCol := 10
+	maxValForId := 10
+	insertShare := 50
+	deleteShare := 50
+	updateShare := 50
+	numQueries := 1000
+	// Wait for schema-tracking to be complete.
+	waitForSchemaTrackingForFkTables(b)
+	for i := 0; i < b.N; i++ {
+		// Clear out all the data to ensure we start with a clean slate.
+		startBenchmark(b)
+		// Create a fuzzer to generate and store a certain set of queries.
+		fz := newFuzzer(1, maxValForId, maxValForCol, insertShare, deleteShare, updateShare, SQLQueries)
+		var queries []string
+		for j := 0; j < numQueries; j++ {
+			genQueries := fz.generateQuery()
+			require.Len(b, genQueries, 1)
+			queries = append(queries, genQueries[0])
+		}
+
+		// Connect to MySQL and run all the queries
+		mysqlConn, err := mysql.Connect(context.Background(), &mysqlParams)
+		require.NoError(b, err)
+		b.Run("MySQL", func(b *testing.B) {
+			runQueries(b, mysqlConn, queries)
+		})
+
+		// Connect to Vitess managed foreign keys keyspace and run all the queries
+		vtConn, err := mysql.Connect(context.Background(), &vtParams)
+		require.NoError(b, err)
+		utils.Exec(b, vtConn, "use `uks`")
+		b.Run("Vitess Managed Foreign Keys", func(b *testing.B) {
+			runQueries(b, vtConn, queries)
+		})
+
+		// Verify that the data in MySQL and Vitess matches. This ensures, we ended up running the same set of queries with the same outputs.
+		for _, table := range fkTables {
+			query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table)
+			resOne, _ := vtConn.ExecuteFetch(query, 10000, true)
+			resTwo, _ := mysqlConn.ExecuteFetch(query, 10000, true)
+			require.True(b, compareResultRows(resOne, resTwo), "Results for %v don't match\nVitess Managed\n%v\nMySQL\n%v", table, resOne, resTwo)
 		}
 	}
 }
+
+func runQueries(t testing.TB, conn *mysql.Conn, queries []string) {
+	for _, query := range queries {
+		_, _ = utils.ExecAllowError(t, conn, query)
+	}
+}

go/test/endtoend/vtgate/foreignkey/main_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package foreignkey
 
 import (
+	"context"
 	_ "embed"
 	"flag"
 	"fmt"
@@ -157,22 +158,7 @@ func start(t *testing.T) (utils.MySQLCompare, func()) {
 	require.NoError(t, err)
 
 	deleteAll := func() {
-		_ = utils.Exec(t, mcmp.VtConn, "use `ks/-80`")
-		tables := []string{"t4", "t3", "t2", "t1", "multicol_tbl2", "multicol_tbl1"}
-		tables = append(tables, fkTables...)
-		for _, table := range tables {
-			_, _ = mcmp.ExecAndIgnore("delete /*+ SET_VAR(foreign_key_checks=OFF) */ from " + table)
-		}
-		_ = utils.Exec(t, mcmp.VtConn, "use `ks/80-`")
-		for _, table := range tables {
-			_, _ = mcmp.ExecAndIgnore("delete /*+ SET_VAR(foreign_key_checks=OFF) */ from " + table)
-		}
-		_ = utils.Exec(t, mcmp.VtConn, "use `uks`")
-		tables = []string{"u_t1", "u_t2", "u_t3"}
-		tables = append(tables, fkTables...)
-		for _, table := range tables {
-			_, _ = mcmp.ExecAndIgnore("delete /*+ SET_VAR(foreign_key_checks=OFF) */ from " + table)
-		}
+		clearOutAllData(t, mcmp.VtConn, mcmp.MySQLConn)
 		_ = utils.Exec(t, mcmp.VtConn, "use `ks`")
 	}
 
@@ -184,3 +170,35 @@ func start(t *testing.T) (utils.MySQLCompare, func()) {
 		cluster.PanicHandler(t)
 	}
 }
+
+func startBenchmark(b *testing.B) {
+	ctx := context.Background()
+	vtConn, err := mysql.Connect(ctx, &vtParams)
+	require.NoError(b, err)
+	mysqlConn, err := mysql.Connect(ctx, &mysqlParams)
+	require.NoError(b, err)
+
+	clearOutAllData(b, vtConn, mysqlConn)
+}
+
+func clearOutAllData(t testing.TB, vtConn *mysql.Conn, mysqlConn *mysql.Conn) {
+	_ = utils.Exec(t, vtConn, "use `ks/-80`")
+	tables := []string{"t4", "t3", "t2", "t1", "multicol_tbl2", "multicol_tbl1"}
+	tables = append(tables, fkTables...)
+	for _, table := range tables {
+		_, _ = utils.ExecAllowError(t, vtConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table)
+		_, _ = utils.ExecAllowError(t, mysqlConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table)
+	}
+	_ = utils.Exec(t, vtConn, "use `ks/80-`")
+	for _, table := range tables {
+		_, _ = utils.ExecAllowError(t, vtConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table)
+		_, _ = utils.ExecAllowError(t, mysqlConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table)
+	}
+	_ = utils.Exec(t, vtConn, "use `uks`")
+	tables = []string{"u_t1", "u_t2", "u_t3"}
+	tables = append(tables, fkTables...)
+	for _, table := range tables {
+		_, _ = utils.ExecAllowError(t, vtConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table)
+		_, _ = utils.ExecAllowError(t, mysqlConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table)
+	}
+}

go/test/endtoend/vtgate/foreignkey/utils_test.go

@@ -20,6 +20,7 @@ import (
 	"database/sql"
 	"fmt"
 	"math/rand"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -75,7 +76,7 @@ func convertIntValueToString(value int) string {
 
 // waitForSchemaTrackingForFkTables waits for schema tracking to have run and seen the tables used
 // for foreign key tests.
-func waitForSchemaTrackingForFkTables(t *testing.T) {
+func waitForSchemaTrackingForFkTables(t testing.TB) {
 	err := utils.WaitForColumn(t, clusterInstance.VtgateProcess, shardedKs, "fk_t1", "col")
 	require.NoError(t, err)
 	err = utils.WaitForColumn(t, clusterInstance.VtgateProcess, shardedKs, "fk_t18", "col")
@@ -238,7 +239,7 @@ func verifyDataIsCorrect(t *testing.T, mcmp utils.MySQLCompare, concurrency int)
 }
 
 // verifyDataMatches verifies that the two list of results are the same.
-func verifyDataMatches(t *testing.T, resOne []*sqltypes.Result, resTwo []*sqltypes.Result) {
+func verifyDataMatches(t testing.TB, resOne []*sqltypes.Result, resTwo []*sqltypes.Result) {
 	require.EqualValues(t, len(resTwo), len(resOne), "Res 1 - %v, Res 2 - %v", resOne, resTwo)
 	for idx, resultOne := range resOne {
 		resultTwo := resTwo[idx]
@@ -256,3 +257,22 @@ func collectFkTablesState(conn *mysql.Conn) []*sqltypes.Result {
 	}
 	return tablesData
 }
+
+func validateReplication(t *testing.T) {
+	for _, keyspace := range clusterInstance.Keyspaces {
+		for _, shard := range keyspace.Shards {
+			for _, vttablet := range shard.Vttablets {
+				if vttablet.Type != "primary" {
+					checkReplicationHealthy(t, vttablet)
+				}
+			}
+		}
+	}
+}
+
+// compareResultRows compares the rows of the two results provided.
+func compareResultRows(resOne *sqltypes.Result, resTwo *sqltypes.Result) bool {
+	return slices.EqualFunc(resOne.Rows, resTwo.Rows, func(a, b sqltypes.Row) bool {
+		return sqltypes.RowEqual(a, b)
+	})
+}