diff --git a/go/test/endtoend/vtgate/queries/aggregation/main_test.go b/go/test/endtoend/vtgate/queries/aggregation/main_test.go index 02013a9b0e2..854c64a9b75 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/main_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/derived/main_test.go b/go/test/endtoend/vtgate/queries/derived/main_test.go index 3b44811f95c..072eb9d2c9e 100644 --- a/go/test/endtoend/vtgate/queries/derived/main_test.go +++ b/go/test/endtoend/vtgate/queries/derived/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/foundrows/main_test.go b/go/test/endtoend/vtgate/queries/foundrows/main_test.go index 8f992863008..dd9ad7fa33f 100644 --- a/go/test/endtoend/vtgate/queries/foundrows/main_test.go +++ b/go/test/endtoend/vtgate/queries/foundrows/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go b/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go index 0be8a50b328..d0b89d56f1d 100644 --- a/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go +++ b/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go @@ -21,14 +21,12 @@ import ( "fmt" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (utils.MySQLCompare, func()) { diff --git a/go/test/endtoend/vtgate/queries/informationschema/main_test.go b/go/test/endtoend/vtgate/queries/informationschema/main_test.go index 06c5b188d18..65a1f3e7d24 100644 --- a/go/test/endtoend/vtgate/queries/informationschema/main_test.go +++ b/go/test/endtoend/vtgate/queries/informationschema/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go b/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go index c385941502a..2cb79a20f91 100644 --- a/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go +++ b/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go @@ -25,12 +25,10 @@ import ( "github.com/stretchr/testify/require" - "vitess.io/vitess/go/test/endtoend/utils" - - "vitess.io/vitess/go/vt/vtgate/planbuilder" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/vt/vtgate/planbuilder" ) var ( diff --git a/go/test/endtoend/vtgate/queries/misc/main_test.go b/go/test/endtoend/vtgate/queries/misc/main_test.go index a3858284884..bc0a0c20bc8 100644 --- a/go/test/endtoend/vtgate/queries/misc/main_test.go +++ b/go/test/endtoend/vtgate/queries/misc/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/normalize/normalize_test.go b/go/test/endtoend/vtgate/queries/normalize/normalize_test.go index b6495443a8e..772ee1a60ac 100644 --- a/go/test/endtoend/vtgate/queries/normalize/normalize_test.go +++ b/go/test/endtoend/vtgate/queries/normalize/normalize_test.go @@ -28,9 +28,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/utils" ) func TestNormalizeAllFields(t *testing.T) { diff --git a/go/test/endtoend/vtgate/queries/orderby/main_test.go b/go/test/endtoend/vtgate/queries/orderby/main_test.go index 9f18377ee3f..7cf2515bde7 100644 --- a/go/test/endtoend/vtgate/queries/orderby/main_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/orderby/orderby_test.go b/go/test/endtoend/vtgate/queries/orderby/orderby_test.go index 43f800ee24c..864faae36ba 100644 --- a/go/test/endtoend/vtgate/queries/orderby/orderby_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/orderby_test.go @@ -19,11 +19,10 @@ package orderby import ( "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (utils.MySQLCompare, func()) { diff --git a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/main_test.go b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/main_test.go index 00221e9c9f3..0f7929e0514 100644 --- a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/main_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go index a20c7ad54c6..72191ba847c 100644 --- a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go @@ -19,11 +19,10 @@ package orderby import ( "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (utils.MySQLCompare, func()) { diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/dept_insert.sql b/go/test/endtoend/vtgate/queries/query_fuzzing/dept_insert.sql new file mode 100644 index 00000000000..416c27287a1 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/dept_insert.sql @@ -0,0 +1,5 @@ +INSERT INTO dept(deptno, dname, loc) +VALUES ('10', 'ACCOUNTING', 'NEW YORK'), + ('20', 'RESEARCH', 'DALLAS'), + ('30', 'SALES', 'CHICAGO'), + ('40', 'OPERATIONS', 'BOSTON') \ No newline at end of file diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/emp_insert.sql b/go/test/endtoend/vtgate/queries/query_fuzzing/emp_insert.sql new file mode 100644 index 00000000000..de44e28bb54 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/emp_insert.sql @@ -0,0 +1,15 @@ +INSERT INTO emp(empno, ename, job, mgr, hiredate, sal, comm, deptno) +VALUES (7369, 'SMITH', 'CLERK', 7902, '1980-12-17', 800, NULL, 20), + (7499, 'ALLEN', 'SALESMAN', 7698, '1981-02-20', 1600, 300, 30), + (7521, 'WARD', 'SALESMAN', 7698, '1981-02-22', 1250, 500, 30), + (7566, 'JONES', 'MANAGER', 7839, '1981-04-02', 2975, NULL, 20), + (7654, 'MARTIN', 'SALESMAN', 7698, '1981-09-28', 1250, 1400, 30), + (7698, 'BLAKE', 'MANAGER', 7839, '1981-05-01', 2850, NULL, 30), + (7782, 'CLARK', 'MANAGER', 7839, '1981-06-09', 2450, NULL, 10), + (7788, 'SCOTT', 'ANALYST', 7566, '1982-12-09', 3000, NULL, 20), + (7839, 'KING', 'PRESIDENT', NULL, '1981-11-17', 5000, NULL, 10), + (7844, 'TURNER', 'SALESMAN', 7698, '1981-09-08', 1500, 0, 30), + (7876, 'ADAMS', 'CLERK', 7788, '1983-01-12', 1100, NULL, 20), + (7900, 'JAMES', 'CLERK', 7698, '1981-12-03', 950, NULL, 30), + (7902, 'FORD', 'ANALYST', 7566, '1981-12-03', 3000, NULL, 20), + (7934, 'MILLER', 'CLERK', 7782, '1982-01-23', 1300, NULL, 10) \ No newline at end of file diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go new file mode 100644 index 00000000000..13aa7f98c91 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go @@ -0,0 +1,130 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package query_fuzzing + +import ( + _ "embed" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql/sqlerror" + "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/vt/sqlparser" +) + +// this test uses the AST defined in the sqlparser package to randomly generate queries + +// if true then execution will always stop on a "must fix" error: a results mismatched or EOF +const stopOnMustFixError = false + +//go:embed emp_insert.sql +var empInsert string + +//go:embed dept_insert.sql +var deptInsert string + +func helperTest(t *testing.T, query string) { + t.Helper() + t.Run(query, func(t *testing.T) { + mcmp, closer := start(t) + defer closer() + + result, err := mcmp.ExecAllowAndCompareError(query) + fmt.Println(result) + fmt.Println(err) + }) +} + +func TestFuzzQueries(t *testing.T) { + mcmp, closer := start(t) + defer closer() + + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) + + // specify the schema (that is defined in schema.sql) + schemaTables := []tableT{ + {tableExpr: sqlparser.NewTableName("emp")}, + {tableExpr: sqlparser.NewTableName("dept")}, + } + schemaTables[0].addColumns([]column{ + {name: "empno", typ: "bigint"}, + {name: "ename", typ: "varchar"}, + {name: "job", typ: "varchar"}, + {name: "mgr", typ: "bigint"}, + {name: "hiredate", typ: "date"}, + {name: "sal", typ: "bigint"}, + {name: "comm", typ: "bigint"}, + {name: "deptno", typ: "bigint"}, + }...) + schemaTables[1].addColumns([]column{ + {name: "deptno", typ: "bigint"}, + {name: "dname", typ: "varchar"}, + {name: "loc", typ: "varchar"}, + }...) + + endBy := time.Now().Add(1 * time.Second) + + var queryCount, queryFailCount int + // continue testing after an error if and only if testFailingQueries is true + for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) { + seed := time.Now().UnixNano() + // genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false) + // qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables) + qg := &maruts{} + query := sqlparser.String(qg.randomQuery()) + t.Run(query, func(t *testing.T) { + _, vtErr := mcmp.ExecAllowAndCompareError(query) + + // this assumes all queries are valid mysql queries + if vtErr != nil { + fmt.Printf("seed: %d\n", seed) + fmt.Println(query) + fmt.Println(vtErr) + + if stopOnMustFixError { + // results mismatched + if strings.Contains(vtErr.Error(), "results mismatched") { + simplified := simplifyResultsMismatchedQuery(t, query) + fmt.Printf("final simplified query: %s\n", simplified) + t.Fatal("failed") + } + // EOF + var sqlError *sqlerror.SQLError + if errors.As(vtErr, &sqlError) && strings.Contains(sqlError.Message, "EOF") { + t.Fatal("failed") + } + } + + // restart the mysql and vitess connections in case something bad happened + closer() + mcmp, closer = start(t) + + fmt.Printf("\n\n\n") + queryFailCount++ + } + queryCount++ + }) + } + fmt.Printf("Queries successfully executed: %d\n", queryCount) + fmt.Printf("Queries failed: %d\n", queryFailCount) +} diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/known_test.go similarity index 71% rename from go/test/endtoend/vtgate/queries/random/random_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/known_test.go index aea43c2f929..e0439571b47 100644 --- a/go/test/endtoend/vtgate/queries/random/random_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/known_test.go @@ -14,70 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( - "fmt" - "math/rand" - "strings" "testing" - "time" - - "vitess.io/vitess/go/mysql/sqlerror" - "vitess.io/vitess/go/vt/sqlparser" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" ) -// this test uses the AST defined in the sqlparser package to randomly generate queries - -// if true then execution will always stop on a "must fix" error: a results mismatched or EOF -const stopOnMustFixError = false - -func start(t *testing.T) (utils.MySQLCompare, func()) { - mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams) - require.NoError(t, err) - - deleteAll := func() { - _, _ = utils.ExecAllowError(t, mcmp.VtConn, "set workload = oltp") - - tables := []string{"emp", "dept"} - for _, table := range tables { - _, _ = mcmp.ExecAndIgnore("delete from " + table) - } - } - - deleteAll() - - // disable only_full_group_by - // mcmp.Exec("set sql_mode=''") - - // insert data - mcmp.Exec("INSERT INTO emp(empno, ename, job, mgr, hiredate, sal, comm, deptno) VALUES (7369,'SMITH','CLERK',7902,'1980-12-17',800,NULL,20), (7499,'ALLEN','SALESMAN',7698,'1981-02-20',1600,300,30), (7521,'WARD','SALESMAN',7698,'1981-02-22',1250,500,30), (7566,'JONES','MANAGER',7839,'1981-04-02',2975,NULL,20), (7654,'MARTIN','SALESMAN',7698,'1981-09-28',1250,1400,30), (7698,'BLAKE','MANAGER',7839,'1981-05-01',2850,NULL,30), (7782,'CLARK','MANAGER',7839,'1981-06-09',2450,NULL,10), (7788,'SCOTT','ANALYST',7566,'1982-12-09',3000,NULL,20), (7839,'KING','PRESIDENT',NULL,'1981-11-17',5000,NULL,10), (7844,'TURNER','SALESMAN',7698,'1981-09-08',1500,0,30), (7876,'ADAMS','CLERK',7788,'1983-01-12',1100,NULL,20), (7900,'JAMES','CLERK',7698,'1981-12-03',950,NULL,30), (7902,'FORD','ANALYST',7566,'1981-12-03',3000,NULL,20), (7934,'MILLER','CLERK',7782,'1982-01-23',1300,NULL,10)") - mcmp.Exec("INSERT INTO dept(deptno, dname, loc) VALUES ('10','ACCOUNTING','NEW YORK'), ('20','RESEARCH','DALLAS'), ('30','SALES','CHICAGO'), ('40','OPERATIONS','BOSTON')") - - return mcmp, func() { - deleteAll() - mcmp.Close() - cluster.PanicHandler(t) - } -} - -func helperTest(t *testing.T, query string) { - t.Helper() - t.Run(query, func(t *testing.T) { - mcmp, closer := start(t) - defer closer() - - result, err := mcmp.ExecAllowAndCompareError(query) - fmt.Println(result) - fmt.Println(err) - }) -} - func TestMustFix(t *testing.T) { t.Skip("Skip CI") @@ -151,16 +97,6 @@ func TestMustFix(t *testing.T) { // EOF helperTest(t, "select count(*) from dept as tbl0, (select count(*) from emp as tbl0, emp as tbl1 limit 18) as tbl1") -} - -func TestKnownFailures(t *testing.T) { - t.Skip("Skip CI") - - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) - - // logs more stuff - // clusterInstance.EnableGeneralLog() // VT13001: [BUG] failed to find the corresponding column helperTest(t, "select tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc") @@ -223,82 +159,8 @@ func TestKnownFailures(t *testing.T) { helperTest(t, "select exists (select 1) as crandom0 from dept as tbl0 group by exists (select 1)") } -func TestRandom(t *testing.T) { - t.Skip("Skip CI; random expressions generate too many failures to properly limit") - - mcmp, closer := start(t) - defer closer() - - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) - - // specify the schema (that is defined in schema.sql) - schemaTables := []tableT{ - {tableExpr: sqlparser.NewTableName("emp")}, - {tableExpr: sqlparser.NewTableName("dept")}, - } - schemaTables[0].addColumns([]column{ - {name: "empno", typ: "bigint"}, - {name: "ename", typ: "varchar"}, - {name: "job", typ: "varchar"}, - {name: "mgr", typ: "bigint"}, - {name: "hiredate", typ: "date"}, - {name: "sal", typ: "bigint"}, - {name: "comm", typ: "bigint"}, - {name: "deptno", typ: "bigint"}, - }...) - schemaTables[1].addColumns([]column{ - {name: "deptno", typ: "bigint"}, - {name: "dname", typ: "varchar"}, - {name: "loc", typ: "varchar"}, - }...) - - endBy := time.Now().Add(1 * time.Second) - - var queryCount, queryFailCount int - // continue testing after an error if and only if testFailingQueries is true - for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) { - seed := time.Now().UnixNano() - genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false) - qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables) - qg.randomQuery() - query := sqlparser.String(qg.stmt) - _, vtErr := mcmp.ExecAllowAndCompareError(query) - - // this assumes all queries are valid mysql queries - if vtErr != nil { - fmt.Printf("seed: %d\n", seed) - fmt.Println(query) - fmt.Println(vtErr) - - if stopOnMustFixError { - // results mismatched - if strings.Contains(vtErr.Error(), "results mismatched") { - simplified := simplifyResultsMismatchedQuery(t, query) - fmt.Printf("final simplified query: %s\n", simplified) - break - } - // EOF - if sqlError, ok := vtErr.(*sqlerror.SQLError); ok && strings.Contains(sqlError.Message, "EOF") { - break - } - } - - // restart the mysql and vitess connections in case something bad happened - closer() - mcmp, closer = start(t) - - fmt.Printf("\n\n\n") - queryFailCount++ - } - queryCount++ - } - fmt.Printf("Queries successfully executed: %d\n", queryCount) - fmt.Printf("Queries failed: %d\n", queryFailCount) -} - // these queries were previously failing and have now been fixed -func TestBuggyQueries(t *testing.T) { +func TestFixedBuggyQueries(t *testing.T) { mcmp, closer := start(t) defer closer() diff --git a/go/test/endtoend/vtgate/queries/random/main_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go similarity index 66% rename from go/test/endtoend/vtgate/queries/random/main_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go index e3256f60796..76c61133c5c 100644 --- a/go/test/endtoend/vtgate/queries/random/main_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go @@ -14,7 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +// query_fuzzing sets up two databases: one is a standard MySQL database and the +// other is a Vitess cluster with identical schemas. +// It inserts the same dataset into both databases. +// The package then generates a large number of queries using query fuzzing, +// executes them on both databases, and verifies that the results match. +package query_fuzzing import ( _ "embed" @@ -23,10 +28,11 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( @@ -63,14 +69,11 @@ func TestMain(m *testing.M) { SchemaSQL: schemaSQL, VSchema: vschema, } - clusterInstance.VtGateExtraArgs = []string{"--schema_change_signal"} - clusterInstance.VtTabletExtraArgs = []string{"--queryserver-config-schema-change-signal"} err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 0, false) if err != nil { return 1 } - clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, "--enable_system_settings=true") // Start vtgate err = clusterInstance.StartVtgate() if err != nil { @@ -92,3 +95,32 @@ func TestMain(m *testing.M) { }() os.Exit(exitCode) } + +func start(t *testing.T) (utils.MySQLCompare, func()) { + mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams) + require.NoError(t, err) + + deleteAll := func() { + _, _ = utils.ExecAllowError(t, mcmp.VtConn, "set workload = oltp") + + tables := []string{"emp", "dept"} + for _, table := range tables { + _, _ = mcmp.ExecAndIgnore("delete from " + table) + } + } + + deleteAll() + + // disable only_full_group_by + mcmp.Exec("set sql_mode=''") + + // insert data + mcmp.Exec(empInsert) + mcmp.Exec(deptInsert) + + return mcmp, func() { + deleteAll() + mcmp.Close() + cluster.PanicHandler(t) + } +} diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go similarity index 98% rename from go/test/endtoend/vtgate/queries/random/query_gen.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go index 3f8fccb05bb..4a66ac87feb 100644 --- a/go/test/endtoend/vtgate/queries/random/query_gen.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( "fmt" @@ -44,20 +44,17 @@ type ( schemaTables []tableT sel *sqlparser.Select } - // queryGenerator generates queries, which can either be unions or select statements queryGenerator struct { stmt sqlparser.SelectStatement selGen *selectGenerator } - column struct { name string // TODO: perhaps remove tableName and always pass columns through a tableT tableName string typ string } - tableT struct { // the tableT struct can be used to represent the schema of a table or a derived table // in the former case tableExpr will be a sqlparser.TableName, in the latter a sqlparser.DerivedTable @@ -67,6 +64,9 @@ type ( alias string cols []column } + queryGen interface { + randomQuery() sqlparser.SelectStatement + } ) var _ sqlparser.ExprGenerator = (*tableT)(nil) @@ -74,6 +74,8 @@ var _ sqlparser.ExprGenerator = (*column)(nil) var _ sqlparser.QueryGenerator = (*selectGenerator)(nil) var _ sqlparser.QueryGenerator = (*queryGenerator)(nil) +var _ queryGen = (*queryGenerator)(nil) + func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, maxTables, maxAggrs, maxGBs int, schemaTables []tableT) *queryGenerator { return &queryGenerator{ selGen: newSelectGenerator(r, genConfig, maxTables, maxAggrs, maxGBs, schemaTables), @@ -196,13 +198,14 @@ func (qg *queryGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGenerat func (sg *selectGenerator) IsQueryGenerator() {} func (qg *queryGenerator) IsQueryGenerator() {} -func (qg *queryGenerator) randomQuery() { +func (qg *queryGenerator) randomQuery() sqlparser.SelectStatement { if qg.selGen.r.Intn(10) < 1 && testFailingQueries { qg.createUnion() } else { qg.selGen.randomSelect() qg.stmt = qg.selGen.sel } + return qg.stmt } // createUnion creates a simple UNION or UNION ALL; no LIMIT or ORDER BY @@ -229,7 +232,6 @@ func (sg *selectGenerator) randomSelect() { sg.genConfig = sg.genConfig.CannotAggregateConfig() sg.sel = &sqlparser.Select{} - sg.sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"}) // select distinct (fails with group by bigint) isDistinct := sg.r.Intn(2) < 1 diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go new file mode 100644 index 00000000000..4b481f00da5 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go @@ -0,0 +1,53 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package query_fuzzing + +import ( + "math/rand" + + "vitess.io/vitess/go/vt/sqlparser" +) + +type ( + // maruts is a query generator that generates random queries to be used in fuzzing + maruts struct { + cfg config + } + + config struct { + r *rand.Rand + genConfig sqlparser.ExprGeneratorConfig + maxTables int + maxAggrs int + maxGBs int + schemaTables []tableT + } +) + +func (m *maruts) randomQuery() sqlparser.SelectStatement { + exprGen := sqlparser.NewGenerator(m.cfg.r, 2) + expr := exprGen.Expression(m.cfg.genConfig) + + ae := &sqlparser.AliasedExpr{Expr: expr} + tableExpr := &sqlparser.AliasedTableExpr{Expr: sqlparser.NewTableName("dual")} + return &sqlparser.Select{ + SelectExprs: sqlparser.SelectExprs{ae}, + From: []sqlparser.TableExpr{tableExpr}, + } +} + +var _ queryGen = (*maruts)(nil) diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen_test.go similarity index 98% rename from go/test/endtoend/vtgate/queries/random/query_gen_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/query_gen_test.go index fe8aa6f6492..5941f7e0e9d 100644 --- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( "fmt" diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/random_expr_test.go similarity index 98% rename from go/test/endtoend/vtgate/queries/random/random_expr_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/random_expr_test.go index 450169a8d9f..448ea434cfb 100644 --- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/random_expr_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( "math/rand" diff --git a/go/test/endtoend/vtgate/queries/random/schema.sql b/go/test/endtoend/vtgate/queries/query_fuzzing/schema.sql similarity index 100% rename from go/test/endtoend/vtgate/queries/random/schema.sql rename to go/test/endtoend/vtgate/queries/query_fuzzing/schema.sql diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go similarity index 52% rename from go/test/endtoend/vtgate/queries/random/simplifier_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go index 2be9ef8ab93..48a3979bb61 100644 --- a/go/test/endtoend/vtgate/queries/random/simplifier_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go @@ -14,18 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( "fmt" "strings" "testing" - "vitess.io/vitess/go/test/vschemawrapper" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/test/vschemawrapper" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/planbuilder" "vitess.io/vitess/go/vt/vtgate/simplifier" @@ -35,23 +34,13 @@ import ( func TestSimplifyResultsMismatchedQuery(t *testing.T) { t.Skip("Skip CI") - var queries []string - queries = append(queries, "select (68 - -16) / case false when -45 then 3 when 28 then -43 else -62 end as crandom0 from dept as tbl0, (select distinct not not false and count(*) from emp as tbl0, emp as tbl1 where tbl1.ename) as tbl1 limit 1", - "select distinct case true when 'burro' then 'trout' else 'elf' end < case count(distinct true) when 'bobcat' then 'turkey' else 'penguin' end from dept as tbl0, emp as tbl1 where 'spider'", - "select distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1 where tbl0.deptno and tbl1.comm in (12, tbl0.deptno, case false when 67 then -17 when -78 then -35 end, -76 >> -68)", - "select count(*) + 1 from emp as tbl0 order by count(*) desc", - "select count(2 >> tbl2.mgr), sum(distinct tbl2.empno <=> 15) from emp as tbl0 left join emp as tbl2 on -32", - "select sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1", - "select tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc", - "select tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) = sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8", - "select distinct count(*) as caggr1 from dept as tbl0, emp as tbl1 group by tbl1.sal having max(tbl1.comm) != true", - "select distinct sum(tbl1.loc) as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(tbl1.dname) <= 1", - "select min(tbl0.deptno) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm", - "select count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 1 = 0", - "select count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 'octopus'", - "select distinct 'octopus' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.empno having count(*) = count(*)", - "select max(tbl0.deptno) from dept as tbl0 right join emp as tbl1 on tbl0.deptno = tbl1.empno and tbl0.deptno = tbl1.deptno group by tbl0.deptno", - "select count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal") + // This test minimizes failing queries to their simplest form where the results still mismatch between the databases. + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) + + queries := []string{ + "select (0 & 0) div min(tbl0.deptno) from emp as tbl0", + } for _, query := range queries { var simplified string @@ -77,14 +66,7 @@ func simplifyResultsMismatchedQuery(t *testing.T, query string) string { defer closer() _, err := mcmp.ExecAllowAndCompareError(query) - if err == nil { - t.Fatalf("query (%s) does not error", query) - } else if !strings.Contains(err.Error(), "mismatched") { - t.Fatalf("query (%s) does not error with results mismatched\nError: %v", query, err) - } - - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) + require.ErrorContainsf(t, err, "mismatched", "query (%s) does not error with results mismatched", query) formal, err := vindexes.LoadFormal("svschema.json") require.NoError(t, err) diff --git a/go/test/endtoend/vtgate/queries/random/svschema.json b/go/test/endtoend/vtgate/queries/query_fuzzing/svschema.json similarity index 100% rename from go/test/endtoend/vtgate/queries/random/svschema.json rename to go/test/endtoend/vtgate/queries/query_fuzzing/svschema.json diff --git a/go/test/endtoend/vtgate/queries/random/vschema.json b/go/test/endtoend/vtgate/queries/query_fuzzing/vschema.json similarity index 100% rename from go/test/endtoend/vtgate/queries/random/vschema.json rename to go/test/endtoend/vtgate/queries/query_fuzzing/vschema.json diff --git a/go/test/endtoend/vtgate/queries/reference/main_test.go b/go/test/endtoend/vtgate/queries/reference/main_test.go index 4c9440ca4ff..591cdc5c548 100644 --- a/go/test/endtoend/vtgate/queries/reference/main_test.go +++ b/go/test/endtoend/vtgate/queries/reference/main_test.go @@ -25,11 +25,9 @@ import ( "time" "vitess.io/vitess/go/mysql" - + "vitess.io/vitess/go/test/endtoend/cluster" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/vtgate/vtgateconn" - - "vitess.io/vitess/go/test/endtoend/cluster" ) var ( diff --git a/go/test/endtoend/vtgate/queries/reference/reference_test.go b/go/test/endtoend/vtgate/queries/reference/reference_test.go index 9c2508a889f..cc0c2efbadb 100644 --- a/go/test/endtoend/vtgate/queries/reference/reference_test.go +++ b/go/test/endtoend/vtgate/queries/reference/reference_test.go @@ -23,9 +23,8 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (*mysql.Conn, func()) { diff --git a/go/test/endtoend/vtgate/queries/subquery/main_test.go b/go/test/endtoend/vtgate/queries/subquery/main_test.go index 9eaf3b4caa0..289b051736e 100644 --- a/go/test/endtoend/vtgate/queries/subquery/main_test.go +++ b/go/test/endtoend/vtgate/queries/subquery/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/timeout/main_test.go b/go/test/endtoend/vtgate/queries/timeout/main_test.go index d71dc55ef46..93ca68bfd39 100644 --- a/go/test/endtoend/vtgate/queries/timeout/main_test.go +++ b/go/test/endtoend/vtgate/queries/timeout/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/union/union_test.go b/go/test/endtoend/vtgate/queries/union/union_test.go index d382d039f02..9dc896a9d1e 100644 --- a/go/test/endtoend/vtgate/queries/union/union_test.go +++ b/go/test/endtoend/vtgate/queries/union/union_test.go @@ -19,12 +19,11 @@ package union import ( "testing" - "vitess.io/vitess/go/test/endtoend/cluster" - - "vitess.io/vitess/go/test/endtoend/utils" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (utils.MySQLCompare, func()) { diff --git a/go/test/endtoend/vtgate/queries/vexplain/main_test.go b/go/test/endtoend/vtgate/queries/vexplain/main_test.go index c1c401bc573..7565b0d2177 100644 --- a/go/test/endtoend/vtgate/queries/vexplain/main_test.go +++ b/go/test/endtoend/vtgate/queries/vexplain/main_test.go @@ -22,10 +22,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/vt/vtgate/planbuilder" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/vt/vtgate/planbuilder" ) var ( diff --git a/go/test/endtoend/vtgate/queries/vexplain/vexplain_test.go b/go/test/endtoend/vtgate/queries/vexplain/vexplain_test.go index ed43d57b578..853b4b85dbf 100644 --- a/go/test/endtoend/vtgate/queries/vexplain/vexplain_test.go +++ b/go/test/endtoend/vtgate/queries/vexplain/vexplain_test.go @@ -22,10 +22,9 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" - - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/utils" ) diff --git a/go/vt/vtgate/simplifier/expression_simplifier.go b/go/vt/vtgate/simplifier/expression_simplifier.go index 5582ac3993d..3a15ba6ebdb 100644 --- a/go/vt/vtgate/simplifier/expression_simplifier.go +++ b/go/vt/vtgate/simplifier/expression_simplifier.go @@ -104,6 +104,8 @@ func (s *shrinker) fillQueue() bool { s.queue = append(s.queue, e.Left, e.Right) case *sqlparser.BinaryExpr: s.queue = append(s.queue, e.Left, e.Right) + case *sqlparser.UnaryExpr: + s.queue = append(s.queue, e.Expr) case *sqlparser.BetweenExpr: s.queue = append(s.queue, e.Left, e.From, e.To) case *sqlparser.Literal: @@ -196,6 +198,10 @@ func (s *shrinker) fillQueue() bool { s.queue = append(s.queue, clone) } } + s.queue = append(s.queue, sqlparser.NewIntLiteral("0")) + s.queue = append(s.queue, sqlparser.NewIntLiteral("1")) + s.queue = append(s.queue, &sqlparser.NullVal{}) + case *sqlparser.ColName: // we can try to replace the column with a literal value s.queue = append(s.queue, sqlparser.NewIntLiteral("0")) diff --git a/go/vt/vtgate/simplifier/simplifier.go b/go/vt/vtgate/simplifier/simplifier.go index 522da172557..0e2716c2caf 100644 --- a/go/vt/vtgate/simplifier/simplifier.go +++ b/go/vt/vtgate/simplifier/simplifier.go @@ -111,6 +111,7 @@ func trySimplifyExpressions(in sqlparser.SelectStatement, test func(sqlparser.Se // initially return false, but that made the rewriter prematurely abort, if it was the last selectExpr return true } + log.Errorf("tried and failed removing expression: %s", sqlparser.String(cursor.expr)) cursor.restore() } @@ -124,6 +125,7 @@ func trySimplifyExpressions(in sqlparser.SelectStatement, test func(sqlparser.Se return true } + log.Errorf("failed to simplify expression: %s -> %s", sqlparser.String(cursor.expr), sqlparser.String(expr)) cursor.restore() return false }) diff --git a/test/config.json b/test/config.json index 7aafcaf1a80..6f995d936e7 100644 --- a/test/config.json +++ b/test/config.json @@ -628,7 +628,7 @@ }, "vtgate_queries_random": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/queries/random"], + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/queries/query_fuzzing"], "Command": [], "Manual": false, "Shard": "vtgate_queries",