From b0cc5c8e3e4eb8145a5859db4d76a6c76f74a2ff Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 17 Nov 2023 14:38:17 +0530 Subject: [PATCH] feat: add benchmarking to compare MySQL and vitess performance on the fuzzer generated queries Signed-off-by: Manan Gupta --- go/test/endtoend/utils/utils.go | 2 +- .../vtgate/foreignkey/fk_fuzz_test.go | 59 ++++++++++++++++--- .../endtoend/vtgate/foreignkey/main_test.go | 50 +++++++++++----- .../endtoend/vtgate/foreignkey/utils_test.go | 24 +++++++- 4 files changed, 108 insertions(+), 27 deletions(-) diff --git a/go/test/endtoend/utils/utils.go b/go/test/endtoend/utils/utils.go index 0231cae7baf..8648b7c9bf3 100644 --- a/go/test/endtoend/utils/utils.go +++ b/go/test/endtoend/utils/utils.go @@ -304,7 +304,7 @@ func WaitForTableDeletions(ctx context.Context, t *testing.T, vtgateProcess clus } // WaitForColumn waits for a table's column to be present -func WaitForColumn(t *testing.T, vtgateProcess cluster.VtgateProcess, ks, tbl, col string) error { +func WaitForColumn(t testing.TB, vtgateProcess cluster.VtgateProcess, ks, tbl, col string) error { timeout := time.After(60 * time.Second) for { select { diff --git a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go index 4c84eae2f42..778c04e9687 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go @@ -17,6 +17,7 @@ limitations under the License. package foreignkey import ( + "context" "database/sql" "fmt" "math/rand" @@ -28,6 +29,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/utils" "vitess.io/vitess/go/vt/log" @@ -671,14 +673,55 @@ func TestFkFuzzTest(t *testing.T) { } } -func validateReplication(t *testing.T) { - for _, keyspace := range clusterInstance.Keyspaces { - for _, shard := range keyspace.Shards { - for _, vttablet := range shard.Vttablets { - if vttablet.Type != "primary" { - checkReplicationHealthy(t, vttablet) - } - } +// BenchmarkFkFuzz benchmarks the performance of Vitess unmanaged, Vitess managed and vanilla MySQL performance on a given set of queries generated by the fuzzer. +func BenchmarkFkFuzz(b *testing.B) { + maxValForCol := 10 + maxValForId := 10 + insertShare := 50 + deleteShare := 50 + updateShare := 50 + numQueries := 1000 + // Wait for schema-tracking to be complete. + waitForSchemaTrackingForFkTables(b) + for i := 0; i < b.N; i++ { + // Clear out all the data to ensure we start with a clean slate. + startBenchmark(b) + // Create a fuzzer to generate and store a certain set of queries. + fz := newFuzzer(1, maxValForId, maxValForCol, insertShare, deleteShare, updateShare, SQLQueries) + var queries []string + for j := 0; j < numQueries; j++ { + genQueries := fz.generateQuery() + require.Len(b, genQueries, 1) + queries = append(queries, genQueries[0]) + } + + // Connect to MySQL and run all the queries + mysqlConn, err := mysql.Connect(context.Background(), &mysqlParams) + require.NoError(b, err) + b.Run("MySQL", func(b *testing.B) { + runQueries(b, mysqlConn, queries) + }) + + // Connect to Vitess managed foreign keys keyspace and run all the queries + vtConn, err := mysql.Connect(context.Background(), &vtParams) + require.NoError(b, err) + utils.Exec(b, vtConn, "use `uks`") + b.Run("Vitess Managed Foreign Keys", func(b *testing.B) { + runQueries(b, vtConn, queries) + }) + + // Verify that the data in MySQL and Vitess matches. This ensures, we ended up running the same set of queries with the same outputs. + for _, table := range fkTables { + query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table) + resOne, _ := vtConn.ExecuteFetch(query, 10000, true) + resTwo, _ := mysqlConn.ExecuteFetch(query, 10000, true) + require.True(b, compareResultRows(resOne, resTwo), "Results for %v don't match\nVitess Managed\n%v\nMySQL\n%v", table, resOne, resTwo) } } } + +func runQueries(t testing.TB, conn *mysql.Conn, queries []string) { + for _, query := range queries { + _, _ = utils.ExecAllowError(t, conn, query) + } +} diff --git a/go/test/endtoend/vtgate/foreignkey/main_test.go b/go/test/endtoend/vtgate/foreignkey/main_test.go index 3ce449f893f..5b4d1768081 100644 --- a/go/test/endtoend/vtgate/foreignkey/main_test.go +++ b/go/test/endtoend/vtgate/foreignkey/main_test.go @@ -17,6 +17,7 @@ limitations under the License. package foreignkey import ( + "context" _ "embed" "flag" "fmt" @@ -157,22 +158,7 @@ func start(t *testing.T) (utils.MySQLCompare, func()) { require.NoError(t, err) deleteAll := func() { - _ = utils.Exec(t, mcmp.VtConn, "use `ks/-80`") - tables := []string{"t4", "t3", "t2", "t1", "multicol_tbl2", "multicol_tbl1"} - tables = append(tables, fkTables...) - for _, table := range tables { - _, _ = mcmp.ExecAndIgnore("delete /*+ SET_VAR(foreign_key_checks=OFF) */ from " + table) - } - _ = utils.Exec(t, mcmp.VtConn, "use `ks/80-`") - for _, table := range tables { - _, _ = mcmp.ExecAndIgnore("delete /*+ SET_VAR(foreign_key_checks=OFF) */ from " + table) - } - _ = utils.Exec(t, mcmp.VtConn, "use `uks`") - tables = []string{"u_t1", "u_t2", "u_t3"} - tables = append(tables, fkTables...) - for _, table := range tables { - _, _ = mcmp.ExecAndIgnore("delete /*+ SET_VAR(foreign_key_checks=OFF) */ from " + table) - } + clearOutAllData(t, mcmp.VtConn, mcmp.MySQLConn) _ = utils.Exec(t, mcmp.VtConn, "use `ks`") } @@ -184,3 +170,35 @@ func start(t *testing.T) (utils.MySQLCompare, func()) { cluster.PanicHandler(t) } } + +func startBenchmark(b *testing.B) { + ctx := context.Background() + vtConn, err := mysql.Connect(ctx, &vtParams) + require.NoError(b, err) + mysqlConn, err := mysql.Connect(ctx, &mysqlParams) + require.NoError(b, err) + + clearOutAllData(b, vtConn, mysqlConn) +} + +func clearOutAllData(t testing.TB, vtConn *mysql.Conn, mysqlConn *mysql.Conn) { + _ = utils.Exec(t, vtConn, "use `ks/-80`") + tables := []string{"t4", "t3", "t2", "t1", "multicol_tbl2", "multicol_tbl1"} + tables = append(tables, fkTables...) + for _, table := range tables { + _, _ = utils.ExecAllowError(t, vtConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table) + _, _ = utils.ExecAllowError(t, mysqlConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table) + } + _ = utils.Exec(t, vtConn, "use `ks/80-`") + for _, table := range tables { + _, _ = utils.ExecAllowError(t, vtConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table) + _, _ = utils.ExecAllowError(t, mysqlConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table) + } + _ = utils.Exec(t, vtConn, "use `uks`") + tables = []string{"u_t1", "u_t2", "u_t3"} + tables = append(tables, fkTables...) + for _, table := range tables { + _, _ = utils.ExecAllowError(t, vtConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table) + _, _ = utils.ExecAllowError(t, mysqlConn, "delete /*+ SET_VAR(foreign_key_checks=OFF) */ from "+table) + } +} diff --git a/go/test/endtoend/vtgate/foreignkey/utils_test.go b/go/test/endtoend/vtgate/foreignkey/utils_test.go index b97e57d685c..a0402d9465b 100644 --- a/go/test/endtoend/vtgate/foreignkey/utils_test.go +++ b/go/test/endtoend/vtgate/foreignkey/utils_test.go @@ -20,6 +20,7 @@ import ( "database/sql" "fmt" "math/rand" + "slices" "strings" "testing" "time" @@ -75,7 +76,7 @@ func convertIntValueToString(value int) string { // waitForSchemaTrackingForFkTables waits for schema tracking to have run and seen the tables used // for foreign key tests. -func waitForSchemaTrackingForFkTables(t *testing.T) { +func waitForSchemaTrackingForFkTables(t testing.TB) { err := utils.WaitForColumn(t, clusterInstance.VtgateProcess, shardedKs, "fk_t1", "col") require.NoError(t, err) err = utils.WaitForColumn(t, clusterInstance.VtgateProcess, shardedKs, "fk_t18", "col") @@ -238,7 +239,7 @@ func verifyDataIsCorrect(t *testing.T, mcmp utils.MySQLCompare, concurrency int) } // verifyDataMatches verifies that the two list of results are the same. -func verifyDataMatches(t *testing.T, resOne []*sqltypes.Result, resTwo []*sqltypes.Result) { +func verifyDataMatches(t testing.TB, resOne []*sqltypes.Result, resTwo []*sqltypes.Result) { require.EqualValues(t, len(resTwo), len(resOne), "Res 1 - %v, Res 2 - %v", resOne, resTwo) for idx, resultOne := range resOne { resultTwo := resTwo[idx] @@ -256,3 +257,22 @@ func collectFkTablesState(conn *mysql.Conn) []*sqltypes.Result { } return tablesData } + +func validateReplication(t *testing.T) { + for _, keyspace := range clusterInstance.Keyspaces { + for _, shard := range keyspace.Shards { + for _, vttablet := range shard.Vttablets { + if vttablet.Type != "primary" { + checkReplicationHealthy(t, vttablet) + } + } + } + } +} + +// compareResultRows compares the rows of the two results provided. +func compareResultRows(resOne *sqltypes.Result, resTwo *sqltypes.Result) bool { + return slices.EqualFunc(resOne.Rows, resTwo.Rows, func(a, b sqltypes.Row) bool { + return sqltypes.RowEqual(a, b) + }) +}