From f5965d866d2e254a7c8869ee12d9fa543d75ce4c Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 24 Nov 2023 10:48:29 +0100 Subject: [PATCH 1/5] query fuzzing: test cleaning and renaming Signed-off-by: Andres Taylor --- .../queries/query_fuzzing/dept_insert.sql | 5 +++++ .../vtgate/queries/query_fuzzing/emp_insert.sql | 15 +++++++++++++++ .../fuzzing_test.go} | 17 ++++++++++++----- .../{random => query_fuzzing}/main_test.go | 10 ++++++---- .../{random => query_fuzzing}/query_gen.go | 3 +-- .../{random => query_fuzzing}/query_gen_test.go | 2 +- .../random_expr_test.go | 2 +- .../{random => query_fuzzing}/schema.sql | 0 .../simplifier_test.go | 2 +- .../{random => query_fuzzing}/svschema.json | 0 .../{random => query_fuzzing}/vschema.json | 0 test/config.json | 2 +- 12 files changed, 43 insertions(+), 15 deletions(-) create mode 100644 go/test/endtoend/vtgate/queries/query_fuzzing/dept_insert.sql create mode 100644 go/test/endtoend/vtgate/queries/query_fuzzing/emp_insert.sql rename go/test/endtoend/vtgate/queries/{random/random_test.go => query_fuzzing/fuzzing_test.go} (93%) rename go/test/endtoend/vtgate/queries/{random => query_fuzzing}/main_test.go (84%) rename go/test/endtoend/vtgate/queries/{random => query_fuzzing}/query_gen.go (99%) rename go/test/endtoend/vtgate/queries/{random => query_fuzzing}/query_gen_test.go (98%) rename go/test/endtoend/vtgate/queries/{random => query_fuzzing}/random_expr_test.go (98%) rename go/test/endtoend/vtgate/queries/{random => query_fuzzing}/schema.sql (100%) rename go/test/endtoend/vtgate/queries/{random => query_fuzzing}/simplifier_test.go (99%) rename go/test/endtoend/vtgate/queries/{random => query_fuzzing}/svschema.json (100%) rename go/test/endtoend/vtgate/queries/{random => query_fuzzing}/vschema.json (100%) diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/dept_insert.sql b/go/test/endtoend/vtgate/queries/query_fuzzing/dept_insert.sql new file mode 100644 index 00000000000..416c27287a1 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/dept_insert.sql @@ -0,0 +1,5 @@ +INSERT INTO dept(deptno, dname, loc) +VALUES ('10', 'ACCOUNTING', 'NEW YORK'), + ('20', 'RESEARCH', 'DALLAS'), + ('30', 'SALES', 'CHICAGO'), + ('40', 'OPERATIONS', 'BOSTON') \ No newline at end of file diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/emp_insert.sql b/go/test/endtoend/vtgate/queries/query_fuzzing/emp_insert.sql new file mode 100644 index 00000000000..de44e28bb54 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/emp_insert.sql @@ -0,0 +1,15 @@ +INSERT INTO emp(empno, ename, job, mgr, hiredate, sal, comm, deptno) +VALUES (7369, 'SMITH', 'CLERK', 7902, '1980-12-17', 800, NULL, 20), + (7499, 'ALLEN', 'SALESMAN', 7698, '1981-02-20', 1600, 300, 30), + (7521, 'WARD', 'SALESMAN', 7698, '1981-02-22', 1250, 500, 30), + (7566, 'JONES', 'MANAGER', 7839, '1981-04-02', 2975, NULL, 20), + (7654, 'MARTIN', 'SALESMAN', 7698, '1981-09-28', 1250, 1400, 30), + (7698, 'BLAKE', 'MANAGER', 7839, '1981-05-01', 2850, NULL, 30), + (7782, 'CLARK', 'MANAGER', 7839, '1981-06-09', 2450, NULL, 10), + (7788, 'SCOTT', 'ANALYST', 7566, '1982-12-09', 3000, NULL, 20), + (7839, 'KING', 'PRESIDENT', NULL, '1981-11-17', 5000, NULL, 10), + (7844, 'TURNER', 'SALESMAN', 7698, '1981-09-08', 1500, 0, 30), + (7876, 'ADAMS', 'CLERK', 7788, '1983-01-12', 1100, NULL, 20), + (7900, 'JAMES', 'CLERK', 7698, '1981-12-03', 950, NULL, 30), + (7902, 'FORD', 'ANALYST', 7566, '1981-12-03', 3000, NULL, 20), + (7934, 'MILLER', 'CLERK', 7782, '1982-01-23', 1300, NULL, 10) \ No newline at end of file diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go similarity index 93% rename from go/test/endtoend/vtgate/queries/random/random_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go index aea43c2f929..6ddb5b216b0 100644 --- a/go/test/endtoend/vtgate/queries/random/random_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( + _ "embed" "fmt" "math/rand" "strings" @@ -37,6 +38,12 @@ import ( // if true then execution will always stop on a "must fix" error: a results mismatched or EOF const stopOnMustFixError = false +//go:embed emp_insert.sql +var empInsert string + +//go:embed dept_insert.sql +var deptInsert string + func start(t *testing.T) (utils.MySQLCompare, func()) { mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams) require.NoError(t, err) @@ -53,11 +60,11 @@ func start(t *testing.T) (utils.MySQLCompare, func()) { deleteAll() // disable only_full_group_by - // mcmp.Exec("set sql_mode=''") + mcmp.Exec("set sql_mode=''") // insert data - mcmp.Exec("INSERT INTO emp(empno, ename, job, mgr, hiredate, sal, comm, deptno) VALUES (7369,'SMITH','CLERK',7902,'1980-12-17',800,NULL,20), (7499,'ALLEN','SALESMAN',7698,'1981-02-20',1600,300,30), (7521,'WARD','SALESMAN',7698,'1981-02-22',1250,500,30), (7566,'JONES','MANAGER',7839,'1981-04-02',2975,NULL,20), (7654,'MARTIN','SALESMAN',7698,'1981-09-28',1250,1400,30), (7698,'BLAKE','MANAGER',7839,'1981-05-01',2850,NULL,30), (7782,'CLARK','MANAGER',7839,'1981-06-09',2450,NULL,10), (7788,'SCOTT','ANALYST',7566,'1982-12-09',3000,NULL,20), (7839,'KING','PRESIDENT',NULL,'1981-11-17',5000,NULL,10), (7844,'TURNER','SALESMAN',7698,'1981-09-08',1500,0,30), (7876,'ADAMS','CLERK',7788,'1983-01-12',1100,NULL,20), (7900,'JAMES','CLERK',7698,'1981-12-03',950,NULL,30), (7902,'FORD','ANALYST',7566,'1981-12-03',3000,NULL,20), (7934,'MILLER','CLERK',7782,'1982-01-23',1300,NULL,10)") - mcmp.Exec("INSERT INTO dept(deptno, dname, loc) VALUES ('10','ACCOUNTING','NEW YORK'), ('20','RESEARCH','DALLAS'), ('30','SALES','CHICAGO'), ('40','OPERATIONS','BOSTON')") + mcmp.Exec(empInsert) + mcmp.Exec(deptInsert) return mcmp, func() { deleteAll() @@ -223,7 +230,7 @@ func TestKnownFailures(t *testing.T) { helperTest(t, "select exists (select 1) as crandom0 from dept as tbl0 group by exists (select 1)") } -func TestRandom(t *testing.T) { +func TestFuzzQueries(t *testing.T) { t.Skip("Skip CI; random expressions generate too many failures to properly limit") mcmp, closer := start(t) diff --git a/go/test/endtoend/vtgate/queries/random/main_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go similarity index 84% rename from go/test/endtoend/vtgate/queries/random/main_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go index e3256f60796..641461737a4 100644 --- a/go/test/endtoend/vtgate/queries/random/main_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go @@ -14,7 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +// query_fuzzing sets up two databases: one is a standard MySQL database and the +// other is a Vitess cluster with identical schemas. +// It inserts the same dataset into both databases. +// The package then generates a large number of queries using query fuzzing, +// executes them on both databases, and verifies that the results match. +package query_fuzzing import ( _ "embed" @@ -63,14 +68,11 @@ func TestMain(m *testing.M) { SchemaSQL: schemaSQL, VSchema: vschema, } - clusterInstance.VtGateExtraArgs = []string{"--schema_change_signal"} - clusterInstance.VtTabletExtraArgs = []string{"--queryserver-config-schema-change-signal"} err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 0, false) if err != nil { return 1 } - clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, "--enable_system_settings=true") // Start vtgate err = clusterInstance.StartVtgate() if err != nil { diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go similarity index 99% rename from go/test/endtoend/vtgate/queries/random/query_gen.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go index 3f8fccb05bb..8b665031364 100644 --- a/go/test/endtoend/vtgate/queries/random/query_gen.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( "fmt" @@ -229,7 +229,6 @@ func (sg *selectGenerator) randomSelect() { sg.genConfig = sg.genConfig.CannotAggregateConfig() sg.sel = &sqlparser.Select{} - sg.sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"}) // select distinct (fails with group by bigint) isDistinct := sg.r.Intn(2) < 1 diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen_test.go similarity index 98% rename from go/test/endtoend/vtgate/queries/random/query_gen_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/query_gen_test.go index fe8aa6f6492..5941f7e0e9d 100644 --- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( "fmt" diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/random_expr_test.go similarity index 98% rename from go/test/endtoend/vtgate/queries/random/random_expr_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/random_expr_test.go index 450169a8d9f..448ea434cfb 100644 --- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/random_expr_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( "math/rand" diff --git a/go/test/endtoend/vtgate/queries/random/schema.sql b/go/test/endtoend/vtgate/queries/query_fuzzing/schema.sql similarity index 100% rename from go/test/endtoend/vtgate/queries/random/schema.sql rename to go/test/endtoend/vtgate/queries/query_fuzzing/schema.sql diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go similarity index 99% rename from go/test/endtoend/vtgate/queries/random/simplifier_test.go rename to go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go index 2be9ef8ab93..67e699b802a 100644 --- a/go/test/endtoend/vtgate/queries/random/simplifier_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package random +package query_fuzzing import ( "fmt" diff --git a/go/test/endtoend/vtgate/queries/random/svschema.json b/go/test/endtoend/vtgate/queries/query_fuzzing/svschema.json similarity index 100% rename from go/test/endtoend/vtgate/queries/random/svschema.json rename to go/test/endtoend/vtgate/queries/query_fuzzing/svschema.json diff --git a/go/test/endtoend/vtgate/queries/random/vschema.json b/go/test/endtoend/vtgate/queries/query_fuzzing/vschema.json similarity index 100% rename from go/test/endtoend/vtgate/queries/random/vschema.json rename to go/test/endtoend/vtgate/queries/query_fuzzing/vschema.json diff --git a/test/config.json b/test/config.json index 7aafcaf1a80..6f995d936e7 100644 --- a/test/config.json +++ b/test/config.json @@ -628,7 +628,7 @@ }, "vtgate_queries_random": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/queries/random"], + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/queries/query_fuzzing"], "Command": [], "Manual": false, "Shard": "vtgate_queries", From 93fb04856eab0f21ca29b91ccf1e88ca2490885c Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 24 Nov 2023 15:21:48 +0100 Subject: [PATCH 2/5] test: imports and cleanup Signed-off-by: Andres Taylor --- .../vtgate/queries/aggregation/main_test.go | 3 +- .../vtgate/queries/derived/main_test.go | 3 +- .../vtgate/queries/foundrows/main_test.go | 3 +- .../informationschema_test.go | 4 +-- .../queries/informationschema/main_test.go | 3 +- .../queries/lookup_queries/main_test.go | 6 ++-- .../endtoend/vtgate/queries/misc/main_test.go | 3 +- .../queries/normalize/normalize_test.go | 3 +- .../vtgate/queries/orderby/main_test.go | 3 +- .../vtgate/queries/orderby/orderby_test.go | 3 +- .../without_schematracker/main_test.go | 3 +- .../without_schematracker/orderby_test.go | 3 +- .../queries/query_fuzzing/fuzzing_test.go | 5 ++- .../vtgate/queries/query_fuzzing/main_test.go | 3 +- .../queries/query_fuzzing/simplifier_test.go | 36 +++++-------------- .../vtgate/queries/reference/main_test.go | 4 +-- .../queries/reference/reference_test.go | 3 +- .../vtgate/queries/subquery/main_test.go | 3 +- .../vtgate/queries/timeout/main_test.go | 3 +- .../vtgate/queries/union/union_test.go | 7 ++-- .../vtgate/queries/vexplain/main_test.go | 3 +- .../vtgate/queries/vexplain/vexplain_test.go | 3 +- .../simplifier/expression_simplifier.go | 6 ++++ go/vt/vtgate/simplifier/simplifier.go | 2 ++ 24 files changed, 42 insertions(+), 76 deletions(-) diff --git a/go/test/endtoend/vtgate/queries/aggregation/main_test.go b/go/test/endtoend/vtgate/queries/aggregation/main_test.go index 02013a9b0e2..854c64a9b75 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/main_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/derived/main_test.go b/go/test/endtoend/vtgate/queries/derived/main_test.go index 3b44811f95c..072eb9d2c9e 100644 --- a/go/test/endtoend/vtgate/queries/derived/main_test.go +++ b/go/test/endtoend/vtgate/queries/derived/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/foundrows/main_test.go b/go/test/endtoend/vtgate/queries/foundrows/main_test.go index 8f992863008..dd9ad7fa33f 100644 --- a/go/test/endtoend/vtgate/queries/foundrows/main_test.go +++ b/go/test/endtoend/vtgate/queries/foundrows/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go b/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go index 0be8a50b328..d0b89d56f1d 100644 --- a/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go +++ b/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go @@ -21,14 +21,12 @@ import ( "fmt" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (utils.MySQLCompare, func()) { diff --git a/go/test/endtoend/vtgate/queries/informationschema/main_test.go b/go/test/endtoend/vtgate/queries/informationschema/main_test.go index 06c5b188d18..65a1f3e7d24 100644 --- a/go/test/endtoend/vtgate/queries/informationschema/main_test.go +++ b/go/test/endtoend/vtgate/queries/informationschema/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go b/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go index c385941502a..2cb79a20f91 100644 --- a/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go +++ b/go/test/endtoend/vtgate/queries/lookup_queries/main_test.go @@ -25,12 +25,10 @@ import ( "github.com/stretchr/testify/require" - "vitess.io/vitess/go/test/endtoend/utils" - - "vitess.io/vitess/go/vt/vtgate/planbuilder" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/vt/vtgate/planbuilder" ) var ( diff --git a/go/test/endtoend/vtgate/queries/misc/main_test.go b/go/test/endtoend/vtgate/queries/misc/main_test.go index a3858284884..bc0a0c20bc8 100644 --- a/go/test/endtoend/vtgate/queries/misc/main_test.go +++ b/go/test/endtoend/vtgate/queries/misc/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/normalize/normalize_test.go b/go/test/endtoend/vtgate/queries/normalize/normalize_test.go index b6495443a8e..772ee1a60ac 100644 --- a/go/test/endtoend/vtgate/queries/normalize/normalize_test.go +++ b/go/test/endtoend/vtgate/queries/normalize/normalize_test.go @@ -28,9 +28,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/utils" ) func TestNormalizeAllFields(t *testing.T) { diff --git a/go/test/endtoend/vtgate/queries/orderby/main_test.go b/go/test/endtoend/vtgate/queries/orderby/main_test.go index 9f18377ee3f..7cf2515bde7 100644 --- a/go/test/endtoend/vtgate/queries/orderby/main_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/orderby/orderby_test.go b/go/test/endtoend/vtgate/queries/orderby/orderby_test.go index 43f800ee24c..864faae36ba 100644 --- a/go/test/endtoend/vtgate/queries/orderby/orderby_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/orderby_test.go @@ -19,11 +19,10 @@ package orderby import ( "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (utils.MySQLCompare, func()) { diff --git a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/main_test.go b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/main_test.go index 00221e9c9f3..0f7929e0514 100644 --- a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/main_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go index a20c7ad54c6..72191ba847c 100644 --- a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go @@ -19,11 +19,10 @@ package orderby import ( "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (utils.MySQLCompare, func()) { diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go index 6ddb5b216b0..3d4fa101206 100644 --- a/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go @@ -24,13 +24,12 @@ import ( "testing" "time" - "vitess.io/vitess/go/mysql/sqlerror" - "vitess.io/vitess/go/vt/sqlparser" - "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/vt/sqlparser" ) // this test uses the AST defined in the sqlparser package to randomly generate queries diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go index 641461737a4..0b54de83df8 100644 --- a/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go @@ -28,10 +28,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go index 67e699b802a..48a3979bb61 100644 --- a/go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/simplifier_test.go @@ -21,11 +21,10 @@ import ( "strings" "testing" - "vitess.io/vitess/go/test/vschemawrapper" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/test/vschemawrapper" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/planbuilder" "vitess.io/vitess/go/vt/vtgate/simplifier" @@ -35,23 +34,13 @@ import ( func TestSimplifyResultsMismatchedQuery(t *testing.T) { t.Skip("Skip CI") - var queries []string - queries = append(queries, "select (68 - -16) / case false when -45 then 3 when 28 then -43 else -62 end as crandom0 from dept as tbl0, (select distinct not not false and count(*) from emp as tbl0, emp as tbl1 where tbl1.ename) as tbl1 limit 1", - "select distinct case true when 'burro' then 'trout' else 'elf' end < case count(distinct true) when 'bobcat' then 'turkey' else 'penguin' end from dept as tbl0, emp as tbl1 where 'spider'", - "select distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1 where tbl0.deptno and tbl1.comm in (12, tbl0.deptno, case false when 67 then -17 when -78 then -35 end, -76 >> -68)", - "select count(*) + 1 from emp as tbl0 order by count(*) desc", - "select count(2 >> tbl2.mgr), sum(distinct tbl2.empno <=> 15) from emp as tbl0 left join emp as tbl2 on -32", - "select sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1", - "select tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc", - "select tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) = sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8", - "select distinct count(*) as caggr1 from dept as tbl0, emp as tbl1 group by tbl1.sal having max(tbl1.comm) != true", - "select distinct sum(tbl1.loc) as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(tbl1.dname) <= 1", - "select min(tbl0.deptno) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm", - "select count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 1 = 0", - "select count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 'octopus'", - "select distinct 'octopus' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.empno having count(*) = count(*)", - "select max(tbl0.deptno) from dept as tbl0 right join emp as tbl1 on tbl0.deptno = tbl1.empno and tbl0.deptno = tbl1.deptno group by tbl0.deptno", - "select count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal") + // This test minimizes failing queries to their simplest form where the results still mismatch between the databases. + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) + + queries := []string{ + "select (0 & 0) div min(tbl0.deptno) from emp as tbl0", + } for _, query := range queries { var simplified string @@ -77,14 +66,7 @@ func simplifyResultsMismatchedQuery(t *testing.T, query string) string { defer closer() _, err := mcmp.ExecAllowAndCompareError(query) - if err == nil { - t.Fatalf("query (%s) does not error", query) - } else if !strings.Contains(err.Error(), "mismatched") { - t.Fatalf("query (%s) does not error with results mismatched\nError: %v", query, err) - } - - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) + require.ErrorContainsf(t, err, "mismatched", "query (%s) does not error with results mismatched", query) formal, err := vindexes.LoadFormal("svschema.json") require.NoError(t, err) diff --git a/go/test/endtoend/vtgate/queries/reference/main_test.go b/go/test/endtoend/vtgate/queries/reference/main_test.go index 4c9440ca4ff..591cdc5c548 100644 --- a/go/test/endtoend/vtgate/queries/reference/main_test.go +++ b/go/test/endtoend/vtgate/queries/reference/main_test.go @@ -25,11 +25,9 @@ import ( "time" "vitess.io/vitess/go/mysql" - + "vitess.io/vitess/go/test/endtoend/cluster" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/vtgate/vtgateconn" - - "vitess.io/vitess/go/test/endtoend/cluster" ) var ( diff --git a/go/test/endtoend/vtgate/queries/reference/reference_test.go b/go/test/endtoend/vtgate/queries/reference/reference_test.go index 9c2508a889f..cc0c2efbadb 100644 --- a/go/test/endtoend/vtgate/queries/reference/reference_test.go +++ b/go/test/endtoend/vtgate/queries/reference/reference_test.go @@ -23,9 +23,8 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (*mysql.Conn, func()) { diff --git a/go/test/endtoend/vtgate/queries/subquery/main_test.go b/go/test/endtoend/vtgate/queries/subquery/main_test.go index 9eaf3b4caa0..289b051736e 100644 --- a/go/test/endtoend/vtgate/queries/subquery/main_test.go +++ b/go/test/endtoend/vtgate/queries/subquery/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/timeout/main_test.go b/go/test/endtoend/vtgate/queries/timeout/main_test.go index d71dc55ef46..93ca68bfd39 100644 --- a/go/test/endtoend/vtgate/queries/timeout/main_test.go +++ b/go/test/endtoend/vtgate/queries/timeout/main_test.go @@ -23,10 +23,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) var ( diff --git a/go/test/endtoend/vtgate/queries/union/union_test.go b/go/test/endtoend/vtgate/queries/union/union_test.go index d382d039f02..9dc896a9d1e 100644 --- a/go/test/endtoend/vtgate/queries/union/union_test.go +++ b/go/test/endtoend/vtgate/queries/union/union_test.go @@ -19,12 +19,11 @@ package union import ( "testing" - "vitess.io/vitess/go/test/endtoend/cluster" - - "vitess.io/vitess/go/test/endtoend/utils" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" ) func start(t *testing.T) (utils.MySQLCompare, func()) { diff --git a/go/test/endtoend/vtgate/queries/vexplain/main_test.go b/go/test/endtoend/vtgate/queries/vexplain/main_test.go index c1c401bc573..7565b0d2177 100644 --- a/go/test/endtoend/vtgate/queries/vexplain/main_test.go +++ b/go/test/endtoend/vtgate/queries/vexplain/main_test.go @@ -22,10 +22,9 @@ import ( "os" "testing" - "vitess.io/vitess/go/vt/vtgate/planbuilder" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/vt/vtgate/planbuilder" ) var ( diff --git a/go/test/endtoend/vtgate/queries/vexplain/vexplain_test.go b/go/test/endtoend/vtgate/queries/vexplain/vexplain_test.go index ed43d57b578..853b4b85dbf 100644 --- a/go/test/endtoend/vtgate/queries/vexplain/vexplain_test.go +++ b/go/test/endtoend/vtgate/queries/vexplain/vexplain_test.go @@ -22,10 +22,9 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" - - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/utils" ) diff --git a/go/vt/vtgate/simplifier/expression_simplifier.go b/go/vt/vtgate/simplifier/expression_simplifier.go index 5582ac3993d..3a15ba6ebdb 100644 --- a/go/vt/vtgate/simplifier/expression_simplifier.go +++ b/go/vt/vtgate/simplifier/expression_simplifier.go @@ -104,6 +104,8 @@ func (s *shrinker) fillQueue() bool { s.queue = append(s.queue, e.Left, e.Right) case *sqlparser.BinaryExpr: s.queue = append(s.queue, e.Left, e.Right) + case *sqlparser.UnaryExpr: + s.queue = append(s.queue, e.Expr) case *sqlparser.BetweenExpr: s.queue = append(s.queue, e.Left, e.From, e.To) case *sqlparser.Literal: @@ -196,6 +198,10 @@ func (s *shrinker) fillQueue() bool { s.queue = append(s.queue, clone) } } + s.queue = append(s.queue, sqlparser.NewIntLiteral("0")) + s.queue = append(s.queue, sqlparser.NewIntLiteral("1")) + s.queue = append(s.queue, &sqlparser.NullVal{}) + case *sqlparser.ColName: // we can try to replace the column with a literal value s.queue = append(s.queue, sqlparser.NewIntLiteral("0")) diff --git a/go/vt/vtgate/simplifier/simplifier.go b/go/vt/vtgate/simplifier/simplifier.go index 522da172557..0e2716c2caf 100644 --- a/go/vt/vtgate/simplifier/simplifier.go +++ b/go/vt/vtgate/simplifier/simplifier.go @@ -111,6 +111,7 @@ func trySimplifyExpressions(in sqlparser.SelectStatement, test func(sqlparser.Se // initially return false, but that made the rewriter prematurely abort, if it was the last selectExpr return true } + log.Errorf("tried and failed removing expression: %s", sqlparser.String(cursor.expr)) cursor.restore() } @@ -124,6 +125,7 @@ func trySimplifyExpressions(in sqlparser.SelectStatement, test func(sqlparser.Se return true } + log.Errorf("failed to simplify expression: %s -> %s", sqlparser.String(cursor.expr), sqlparser.String(expr)) cursor.restore() return false }) From 2656794b390f0f605338c583f0ea7b540820761e Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 28 Nov 2023 11:11:25 +0100 Subject: [PATCH 3/5] test: move test methods around Signed-off-by: Andres Taylor --- .../queries/query_fuzzing/fuzzing_test.go | 220 ------------------ .../queries/query_fuzzing/known_test.go | 205 ++++++++++++++++ .../vtgate/queries/query_fuzzing/main_test.go | 31 +++ 3 files changed, 236 insertions(+), 220 deletions(-) create mode 100644 go/test/endtoend/vtgate/queries/query_fuzzing/known_test.go diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go index 3d4fa101206..6b6f9a06e0e 100644 --- a/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go @@ -27,7 +27,6 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql/sqlerror" - "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" "vitess.io/vitess/go/vt/sqlparser" ) @@ -43,35 +42,6 @@ var empInsert string //go:embed dept_insert.sql var deptInsert string -func start(t *testing.T) (utils.MySQLCompare, func()) { - mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams) - require.NoError(t, err) - - deleteAll := func() { - _, _ = utils.ExecAllowError(t, mcmp.VtConn, "set workload = oltp") - - tables := []string{"emp", "dept"} - for _, table := range tables { - _, _ = mcmp.ExecAndIgnore("delete from " + table) - } - } - - deleteAll() - - // disable only_full_group_by - mcmp.Exec("set sql_mode=''") - - // insert data - mcmp.Exec(empInsert) - mcmp.Exec(deptInsert) - - return mcmp, func() { - deleteAll() - mcmp.Close() - cluster.PanicHandler(t) - } -} - func helperTest(t *testing.T, query string) { t.Helper() t.Run(query, func(t *testing.T) { @@ -84,151 +54,6 @@ func helperTest(t *testing.T, query string) { }) } -func TestMustFix(t *testing.T) { - t.Skip("Skip CI") - - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) - - // results mismatched - helperTest(t, "select distinct case count(*) when 0 then -0 end from emp as tbl0, emp as tbl1 where 0") - - // results mismatched (maybe derived tables) - helperTest(t, "select 0 as crandom0 from dept as tbl0, (select distinct count(*) from emp as tbl1 where 0) as tbl1") - - // results mismatched - helperTest(t, "select distinct case count(distinct true) when 'b' then 't' end from emp as tbl1 where 's'") - - // results mismatched - helperTest(t, "select distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1") - - // mismatched number of columns - helperTest(t, "select count(*) + 0 from emp as tbl0 order by count(*) desc") - - // results mismatched (mismatched types) - helperTest(t, "select count(0 >> 0), sum(distinct tbl2.empno) from emp as tbl0 left join emp as tbl2 on -32") - - // results mismatched (decimals off by a little; evalengine problem) - helperTest(t, "select sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1") - - // EOF - helperTest(t, "select tbl1.dname as cgroup0, tbl1.dname as cgroup1, tbl1.deptno as crandom0 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc") - - // results mismatched - // limit >= 9 works - helperTest(t, "select tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8") - - // results mismatched - helperTest(t, "select distinct count(*) as caggr1 from emp as tbl1 group by tbl1.sal having max(0) != true") - - // results mismatched - helperTest(t, "select distinct 0 as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(0) <= 0") - - // results mismatched - helperTest(t, "select min(0) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm") - - // results mismatched - helperTest(t, "select count(*) as caggr0, 0 as crandom0 from dept as tbl0, emp as tbl1 where 0") - - // results mismatched - helperTest(t, "select count(*) as caggr0, 0 as crandom0 from dept as tbl0, emp as tbl1 where 'o'") - - // similar to previous two - // results mismatched - helperTest(t, "select distinct 'o' as crandom0 from dept as tbl0, emp as tbl1 where 0 having count(*) = count(*)") - - // results mismatched (group by + right join) - // left instead of right works - // swapping tables and predicates and changing to left fails - helperTest(t, "select 0 from dept as tbl0 right join emp as tbl1 on tbl0.deptno = tbl1.empno and tbl0.deptno = tbl1.deptno group by tbl0.deptno") - - // results mismatched (count + right join) - // left instead of right works - // swapping tables and predicates and changing to left fails - helperTest(t, "select count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal") - - // Passes with different errors - // vitess error: EOF - // mysql error: Operand should contain 1 column(s) - helperTest(t, "select 8 < -31 xor (-29, sum((tbl0.deptno, 'wren', 'ostrich')), max(distinct (tbl0.dname, -15, -8))) in ((sum(distinct (tbl0.dname, 'bengal', -10)), 'ant', true)) as caggr0 from dept as tbl0 where tbl0.deptno * (77 - 61)") - - // EOF - helperTest(t, "select tbl1.deptno as cgroup0, tbl1.loc as cgroup1, count(distinct tbl1.loc) as caggr1, tbl1.loc as crandom0 from dept as tbl0, dept as tbl1 group by tbl1.deptno, tbl1.loc") - - // EOF - helperTest(t, "select count(*) from dept as tbl0, (select count(*) from emp as tbl0, emp as tbl1 limit 18) as tbl1") -} - -func TestKnownFailures(t *testing.T) { - t.Skip("Skip CI") - - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) - - // logs more stuff - // clusterInstance.EnableGeneralLog() - - // VT13001: [BUG] failed to find the corresponding column - helperTest(t, "select tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc") - - // vitess error: - // mysql error: Operand should contain 1 column(s) - helperTest(t, "select (count('sheepdog') ^ (-71 % sum(emp.mgr) ^ count('koi')) and count(*), 'fly') from emp, dept") - - // rhs of an In operation should be a tuple - helperTest(t, "select (case when true then min(distinct tbl1.job) else 'bee' end, 'molly') not in (('dane', 0)) as caggr1 from emp as tbl0, emp as tbl1") - - // VT13001: [BUG] in scatter query: complex ORDER BY expression: :vtg1 /* VARCHAR */ - helperTest(t, "select tbl1.job as cgroup0, sum(distinct 'mudfish'), tbl1.job as crandom0 from emp as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job asc limit 8, 1") - - // unsupported: min/max on types that are not comparable is not supported - helperTest(t, "select max(case true when false then 'gnu' when true then 'meerkat' end) as caggr0 from dept as tbl0") - - // vttablet: rpc error: code = InvalidArgument desc = BIGINT UNSIGNED value is out of range in '(-(273) + (-(15) & 124))' - helperTest(t, "select -273 + (-15 & 124) as crandom0 from emp as tbl0, emp as tbl1 where tbl1.sal >= tbl1.mgr") - - // vitess error: - // mysql error: Incorrect DATE value: 'tuna' - helperTest(t, "select min(tbl0.empno) as caggr0 from emp as tbl0 where case 'gator' when false then 314 else 'weevil' end > tbl0.job having min(tbl0.hiredate) <=> 'tuna'") - - // vitess error: - // mysql error: Unknown column 'tbl0.deptno' in 'having clause' - helperTest(t, "select count(*) as caggr0 from dept as tbl0 having tbl0.deptno") - - // only_full_group_by enabled - // vitess error: In aggregated query without GROUP BY, expression #1 of SELECT list contains nonaggregated column 'ks_random.tbl0.EMPNO'; this is incompatible with sql_mode=only_full_group_by - helperTest(t, "select distinct tbl0.empno as cgroup0, count(distinct 56) as caggr0, min('flounder' = 'penguin') as caggr1 from emp as tbl0, (select 'manatee' as crandom0 from dept as tbl0 where -26 limit 2) as tbl2 where 'anteater' like 'catfish' is null and -11 group by tbl0.empno order by tbl0.empno asc, count(distinct 56) asc, min('flounder' = 'penguin') desc") - - // only_full_group_by enabled - // vitess error: - // mysql error: In aggregated query without GROUP BY, expression #1 of SELECT list contains nonaggregated column 'ks_random.tbl0.ENAME'; this is incompatible with sql_mode=only_full_group_by - helperTest(t, "select tbl0.ename, min(tbl0.comm) from emp as tbl0 left join emp as tbl1 on tbl0.empno = tbl1.comm and tbl0.empno = tbl1.empno") - - // only_full_group_by enabled - // vitess error: - // mysql error: Expression #1 of ORDER BY clause is not in SELECT list, references column 'ks_random.tbl2.DNAME' which is not in SELECT list; this is incompatible with DISTINCT - helperTest(t, "select distinct count(*) as caggr0 from dept as tbl2 group by tbl2.dname order by tbl2.dname asc") - - // vttablet: rpc error: code = NotFound desc = Unknown column 'cgroup0' in 'field list' (errno 1054) (sqlstate 42S22) (CallerID: userData1) - helperTest(t, "select tbl1.ename as cgroup0, max(tbl0.comm) as caggr0 from emp as tbl0, emp as tbl1 group by cgroup0") - - // unsupported - // VT12001: unsupported: only one DISTINCT aggregation is allowed in a SELECT: sum(distinct 1) as caggr1 - helperTest(t, "select sum(distinct tbl0.comm) as caggr0, sum(distinct 1) as caggr1 from emp as tbl0 having 'redfish' < 'blowfish'") - - // unsupported - // VT12001: unsupported: in scatter query: aggregation function 'avg(tbl0.deptno)' - helperTest(t, "select avg(tbl0.deptno) from dept as tbl0") - - // unsupported - // VT12001: unsupported: LEFT JOIN with derived tables - helperTest(t, "select -1 as crandom0 from emp as tbl2 left join (select count(*) from dept as tbl1) as tbl3 on 6 != tbl2.deptno") - - // unsupported - // VT12001: unsupported: subqueries in GROUP BY - helperTest(t, "select exists (select 1) as crandom0 from dept as tbl0 group by exists (select 1)") -} - func TestFuzzQueries(t *testing.T) { t.Skip("Skip CI; random expressions generate too many failures to properly limit") @@ -302,48 +127,3 @@ func TestFuzzQueries(t *testing.T) { fmt.Printf("Queries successfully executed: %d\n", queryCount) fmt.Printf("Queries failed: %d\n", queryFailCount) } - -// these queries were previously failing and have now been fixed -func TestBuggyQueries(t *testing.T) { - mcmp, closer := start(t) - defer closer() - - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) - require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) - - mcmp.Exec("select sum(tbl1.sal) as caggr1 from emp as tbl0, emp as tbl1 group by tbl1.ename order by tbl1.ename asc") - mcmp.Exec("select count(*), count(*), count(*) from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.deptno group by tbl1.empno order by tbl1.empno") - mcmp.Exec("select count(tbl0.deptno) from dept as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job limit 3") - mcmp.Exec("select count(*), count(*) from emp as tbl0 group by tbl0.empno order by tbl0.empno") - mcmp.Exec("select distinct count(*), tbl0.loc from dept as tbl0 group by tbl0.loc") - mcmp.Exec("select distinct count(*) from dept as tbl0 group by tbl0.loc") - mcmp.Exec("select sum(tbl1.comm) from emp as tbl0, emp as tbl1") - mcmp.Exec("select tbl1.mgr, tbl1.mgr, count(*) from emp as tbl1 group by tbl1.mgr") - mcmp.Exec("select tbl1.mgr, tbl1.mgr, count(*) from emp as tbl0, emp as tbl1 group by tbl1.mgr") - mcmp.Exec("select count(*), count(*), count(tbl0.comm) from emp as tbl0, emp as tbl1 join dept as tbl2") - mcmp.Exec("select count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0, dept as tbl1") - mcmp.Exec("select count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0") - mcmp.Exec("select min(tbl0.loc) from dept as tbl0") - mcmp.Exec("select tbl1.empno, max(tbl1.job) from dept as tbl0, emp as tbl1 group by tbl1.empno") - mcmp.Exec("select tbl1.ename, max(tbl0.comm) from emp as tbl0, emp as tbl1 group by tbl1.ename") - mcmp.Exec("select tbl0.dname, tbl0.dname, min(tbl0.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl0.dname") - mcmp.Exec("select tbl0.dname, min(tbl1.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl1.dname") - mcmp.Exec("select max(tbl0.hiredate) from emp as tbl0") - mcmp.Exec("select min(tbl0.deptno) as caggr0, count(*) as caggr1 from dept as tbl0 left join dept as tbl1 on tbl1.loc = tbl1.dname") - mcmp.Exec("select count(tbl1.loc) as caggr0 from dept as tbl1 left join dept as tbl2 on tbl1.loc = tbl2.loc where (tbl2.deptno)") - mcmp.Exec("select sum(tbl1.ename), min(tbl0.empno) from emp as tbl0, emp as tbl1 left join dept as tbl2 on tbl1.job = tbl2.loc and tbl1.comm = tbl2.deptno where ('trout') and tbl0.deptno = tbl1.comm") - mcmp.Exec("select distinct max(tbl0.deptno), count(tbl0.job) from emp as tbl0, dept as tbl1 left join dept as tbl2 on tbl1.dname = tbl2.loc and tbl1.dname = tbl2.loc where (tbl2.loc) and tbl0.deptno = tbl1.deptno") - mcmp.Exec("select count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0") - mcmp.Exec("select distinct max(tbl0.dname) as caggr0, 'cattle' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno != tbl1.sal group by tbl1.comm") - mcmp.Exec("select -26 in (tbl2.mgr, -8, tbl0.deptno) as crandom0 from dept as tbl0, emp as tbl1 left join emp as tbl2 on tbl2.ename") - mcmp.Exec("select max(tbl1.dname) as caggr1 from dept as tbl0, dept as tbl1 group by tbl1.dname order by tbl1.dname asc") - mcmp.Exec("select distinct tbl1.hiredate as cgroup0, count(tbl1.mgr) as caggr0 from emp as tbl1 group by tbl1.hiredate, tbl1.ename") - mcmp.Exec("select distinct 347 as crandom0 from emp as tbl0") - mcmp.Exec("select distinct count(*) from dept as tbl0 group by tbl0.deptno") - mcmp.Exec("select count(*) from dept as tbl1 join (select count(*) from emp as tbl0, dept as tbl1 group by tbl1.loc) as tbl2") - mcmp.Exec("select (select count(*) from emp as tbl0) from emp as tbl0") - mcmp.Exec("select count(tbl1.dname) as caggr1 from dept as tbl0 left join dept as tbl1 on tbl1.dname > tbl1.loc where tbl1.loc <=> tbl1.dname group by tbl1.dname order by tbl1.dname asc") - mcmp.Exec("select count(*) from (select count(*) from dept as tbl0) as tbl0") - mcmp.Exec("select count(*), count(*) from (select count(*) from dept as tbl0) as tbl0, dept as tbl1") - mcmp.Exec(`select distinct case max(tbl0.ename) when min(tbl0.job) then 'sole' else count(case when false then -27 when 'gazelle' then tbl0.deptno end) end as caggr0 from emp as tbl0`) -} diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/known_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/known_test.go new file mode 100644 index 00000000000..e0439571b47 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/known_test.go @@ -0,0 +1,205 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package query_fuzzing + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/endtoend/utils" +) + +func TestMustFix(t *testing.T) { + t.Skip("Skip CI") + + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) + + // results mismatched + helperTest(t, "select distinct case count(*) when 0 then -0 end from emp as tbl0, emp as tbl1 where 0") + + // results mismatched (maybe derived tables) + helperTest(t, "select 0 as crandom0 from dept as tbl0, (select distinct count(*) from emp as tbl1 where 0) as tbl1") + + // results mismatched + helperTest(t, "select distinct case count(distinct true) when 'b' then 't' end from emp as tbl1 where 's'") + + // results mismatched + helperTest(t, "select distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1") + + // mismatched number of columns + helperTest(t, "select count(*) + 0 from emp as tbl0 order by count(*) desc") + + // results mismatched (mismatched types) + helperTest(t, "select count(0 >> 0), sum(distinct tbl2.empno) from emp as tbl0 left join emp as tbl2 on -32") + + // results mismatched (decimals off by a little; evalengine problem) + helperTest(t, "select sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1") + + // EOF + helperTest(t, "select tbl1.dname as cgroup0, tbl1.dname as cgroup1, tbl1.deptno as crandom0 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc") + + // results mismatched + // limit >= 9 works + helperTest(t, "select tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8") + + // results mismatched + helperTest(t, "select distinct count(*) as caggr1 from emp as tbl1 group by tbl1.sal having max(0) != true") + + // results mismatched + helperTest(t, "select distinct 0 as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(0) <= 0") + + // results mismatched + helperTest(t, "select min(0) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm") + + // results mismatched + helperTest(t, "select count(*) as caggr0, 0 as crandom0 from dept as tbl0, emp as tbl1 where 0") + + // results mismatched + helperTest(t, "select count(*) as caggr0, 0 as crandom0 from dept as tbl0, emp as tbl1 where 'o'") + + // similar to previous two + // results mismatched + helperTest(t, "select distinct 'o' as crandom0 from dept as tbl0, emp as tbl1 where 0 having count(*) = count(*)") + + // results mismatched (group by + right join) + // left instead of right works + // swapping tables and predicates and changing to left fails + helperTest(t, "select 0 from dept as tbl0 right join emp as tbl1 on tbl0.deptno = tbl1.empno and tbl0.deptno = tbl1.deptno group by tbl0.deptno") + + // results mismatched (count + right join) + // left instead of right works + // swapping tables and predicates and changing to left fails + helperTest(t, "select count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal") + + // Passes with different errors + // vitess error: EOF + // mysql error: Operand should contain 1 column(s) + helperTest(t, "select 8 < -31 xor (-29, sum((tbl0.deptno, 'wren', 'ostrich')), max(distinct (tbl0.dname, -15, -8))) in ((sum(distinct (tbl0.dname, 'bengal', -10)), 'ant', true)) as caggr0 from dept as tbl0 where tbl0.deptno * (77 - 61)") + + // EOF + helperTest(t, "select tbl1.deptno as cgroup0, tbl1.loc as cgroup1, count(distinct tbl1.loc) as caggr1, tbl1.loc as crandom0 from dept as tbl0, dept as tbl1 group by tbl1.deptno, tbl1.loc") + + // EOF + helperTest(t, "select count(*) from dept as tbl0, (select count(*) from emp as tbl0, emp as tbl1 limit 18) as tbl1") + + // VT13001: [BUG] failed to find the corresponding column + helperTest(t, "select tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc") + + // vitess error: + // mysql error: Operand should contain 1 column(s) + helperTest(t, "select (count('sheepdog') ^ (-71 % sum(emp.mgr) ^ count('koi')) and count(*), 'fly') from emp, dept") + + // rhs of an In operation should be a tuple + helperTest(t, "select (case when true then min(distinct tbl1.job) else 'bee' end, 'molly') not in (('dane', 0)) as caggr1 from emp as tbl0, emp as tbl1") + + // VT13001: [BUG] in scatter query: complex ORDER BY expression: :vtg1 /* VARCHAR */ + helperTest(t, "select tbl1.job as cgroup0, sum(distinct 'mudfish'), tbl1.job as crandom0 from emp as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job asc limit 8, 1") + + // unsupported: min/max on types that are not comparable is not supported + helperTest(t, "select max(case true when false then 'gnu' when true then 'meerkat' end) as caggr0 from dept as tbl0") + + // vttablet: rpc error: code = InvalidArgument desc = BIGINT UNSIGNED value is out of range in '(-(273) + (-(15) & 124))' + helperTest(t, "select -273 + (-15 & 124) as crandom0 from emp as tbl0, emp as tbl1 where tbl1.sal >= tbl1.mgr") + + // vitess error: + // mysql error: Incorrect DATE value: 'tuna' + helperTest(t, "select min(tbl0.empno) as caggr0 from emp as tbl0 where case 'gator' when false then 314 else 'weevil' end > tbl0.job having min(tbl0.hiredate) <=> 'tuna'") + + // vitess error: + // mysql error: Unknown column 'tbl0.deptno' in 'having clause' + helperTest(t, "select count(*) as caggr0 from dept as tbl0 having tbl0.deptno") + + // only_full_group_by enabled + // vitess error: In aggregated query without GROUP BY, expression #1 of SELECT list contains nonaggregated column 'ks_random.tbl0.EMPNO'; this is incompatible with sql_mode=only_full_group_by + helperTest(t, "select distinct tbl0.empno as cgroup0, count(distinct 56) as caggr0, min('flounder' = 'penguin') as caggr1 from emp as tbl0, (select 'manatee' as crandom0 from dept as tbl0 where -26 limit 2) as tbl2 where 'anteater' like 'catfish' is null and -11 group by tbl0.empno order by tbl0.empno asc, count(distinct 56) asc, min('flounder' = 'penguin') desc") + + // only_full_group_by enabled + // vitess error: + // mysql error: In aggregated query without GROUP BY, expression #1 of SELECT list contains nonaggregated column 'ks_random.tbl0.ENAME'; this is incompatible with sql_mode=only_full_group_by + helperTest(t, "select tbl0.ename, min(tbl0.comm) from emp as tbl0 left join emp as tbl1 on tbl0.empno = tbl1.comm and tbl0.empno = tbl1.empno") + + // only_full_group_by enabled + // vitess error: + // mysql error: Expression #1 of ORDER BY clause is not in SELECT list, references column 'ks_random.tbl2.DNAME' which is not in SELECT list; this is incompatible with DISTINCT + helperTest(t, "select distinct count(*) as caggr0 from dept as tbl2 group by tbl2.dname order by tbl2.dname asc") + + // vttablet: rpc error: code = NotFound desc = Unknown column 'cgroup0' in 'field list' (errno 1054) (sqlstate 42S22) (CallerID: userData1) + helperTest(t, "select tbl1.ename as cgroup0, max(tbl0.comm) as caggr0 from emp as tbl0, emp as tbl1 group by cgroup0") + + // unsupported + // VT12001: unsupported: only one DISTINCT aggregation is allowed in a SELECT: sum(distinct 1) as caggr1 + helperTest(t, "select sum(distinct tbl0.comm) as caggr0, sum(distinct 1) as caggr1 from emp as tbl0 having 'redfish' < 'blowfish'") + + // unsupported + // VT12001: unsupported: in scatter query: aggregation function 'avg(tbl0.deptno)' + helperTest(t, "select avg(tbl0.deptno) from dept as tbl0") + + // unsupported + // VT12001: unsupported: LEFT JOIN with derived tables + helperTest(t, "select -1 as crandom0 from emp as tbl2 left join (select count(*) from dept as tbl1) as tbl3 on 6 != tbl2.deptno") + + // unsupported + // VT12001: unsupported: subqueries in GROUP BY + helperTest(t, "select exists (select 1) as crandom0 from dept as tbl0 group by exists (select 1)") +} + +// these queries were previously failing and have now been fixed +func TestFixedBuggyQueries(t *testing.T) { + mcmp, closer := start(t) + defer closer() + + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema)) + require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema)) + + mcmp.Exec("select sum(tbl1.sal) as caggr1 from emp as tbl0, emp as tbl1 group by tbl1.ename order by tbl1.ename asc") + mcmp.Exec("select count(*), count(*), count(*) from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.deptno group by tbl1.empno order by tbl1.empno") + mcmp.Exec("select count(tbl0.deptno) from dept as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job limit 3") + mcmp.Exec("select count(*), count(*) from emp as tbl0 group by tbl0.empno order by tbl0.empno") + mcmp.Exec("select distinct count(*), tbl0.loc from dept as tbl0 group by tbl0.loc") + mcmp.Exec("select distinct count(*) from dept as tbl0 group by tbl0.loc") + mcmp.Exec("select sum(tbl1.comm) from emp as tbl0, emp as tbl1") + mcmp.Exec("select tbl1.mgr, tbl1.mgr, count(*) from emp as tbl1 group by tbl1.mgr") + mcmp.Exec("select tbl1.mgr, tbl1.mgr, count(*) from emp as tbl0, emp as tbl1 group by tbl1.mgr") + mcmp.Exec("select count(*), count(*), count(tbl0.comm) from emp as tbl0, emp as tbl1 join dept as tbl2") + mcmp.Exec("select count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0, dept as tbl1") + mcmp.Exec("select count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0") + mcmp.Exec("select min(tbl0.loc) from dept as tbl0") + mcmp.Exec("select tbl1.empno, max(tbl1.job) from dept as tbl0, emp as tbl1 group by tbl1.empno") + mcmp.Exec("select tbl1.ename, max(tbl0.comm) from emp as tbl0, emp as tbl1 group by tbl1.ename") + mcmp.Exec("select tbl0.dname, tbl0.dname, min(tbl0.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl0.dname") + mcmp.Exec("select tbl0.dname, min(tbl1.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl1.dname") + mcmp.Exec("select max(tbl0.hiredate) from emp as tbl0") + mcmp.Exec("select min(tbl0.deptno) as caggr0, count(*) as caggr1 from dept as tbl0 left join dept as tbl1 on tbl1.loc = tbl1.dname") + mcmp.Exec("select count(tbl1.loc) as caggr0 from dept as tbl1 left join dept as tbl2 on tbl1.loc = tbl2.loc where (tbl2.deptno)") + mcmp.Exec("select sum(tbl1.ename), min(tbl0.empno) from emp as tbl0, emp as tbl1 left join dept as tbl2 on tbl1.job = tbl2.loc and tbl1.comm = tbl2.deptno where ('trout') and tbl0.deptno = tbl1.comm") + mcmp.Exec("select distinct max(tbl0.deptno), count(tbl0.job) from emp as tbl0, dept as tbl1 left join dept as tbl2 on tbl1.dname = tbl2.loc and tbl1.dname = tbl2.loc where (tbl2.loc) and tbl0.deptno = tbl1.deptno") + mcmp.Exec("select count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0") + mcmp.Exec("select distinct max(tbl0.dname) as caggr0, 'cattle' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno != tbl1.sal group by tbl1.comm") + mcmp.Exec("select -26 in (tbl2.mgr, -8, tbl0.deptno) as crandom0 from dept as tbl0, emp as tbl1 left join emp as tbl2 on tbl2.ename") + mcmp.Exec("select max(tbl1.dname) as caggr1 from dept as tbl0, dept as tbl1 group by tbl1.dname order by tbl1.dname asc") + mcmp.Exec("select distinct tbl1.hiredate as cgroup0, count(tbl1.mgr) as caggr0 from emp as tbl1 group by tbl1.hiredate, tbl1.ename") + mcmp.Exec("select distinct 347 as crandom0 from emp as tbl0") + mcmp.Exec("select distinct count(*) from dept as tbl0 group by tbl0.deptno") + mcmp.Exec("select count(*) from dept as tbl1 join (select count(*) from emp as tbl0, dept as tbl1 group by tbl1.loc) as tbl2") + mcmp.Exec("select (select count(*) from emp as tbl0) from emp as tbl0") + mcmp.Exec("select count(tbl1.dname) as caggr1 from dept as tbl0 left join dept as tbl1 on tbl1.dname > tbl1.loc where tbl1.loc <=> tbl1.dname group by tbl1.dname order by tbl1.dname asc") + mcmp.Exec("select count(*) from (select count(*) from dept as tbl0) as tbl0") + mcmp.Exec("select count(*), count(*) from (select count(*) from dept as tbl0) as tbl0, dept as tbl1") + mcmp.Exec(`select distinct case max(tbl0.ename) when min(tbl0.job) then 'sole' else count(case when false then -27 when 'gazelle' then tbl0.deptno end) end as caggr0 from emp as tbl0`) +} diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go index 0b54de83df8..76c61133c5c 100644 --- a/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/main_test.go @@ -28,6 +28,8 @@ import ( "os" "testing" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" @@ -93,3 +95,32 @@ func TestMain(m *testing.M) { }() os.Exit(exitCode) } + +func start(t *testing.T) (utils.MySQLCompare, func()) { + mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams) + require.NoError(t, err) + + deleteAll := func() { + _, _ = utils.ExecAllowError(t, mcmp.VtConn, "set workload = oltp") + + tables := []string{"emp", "dept"} + for _, table := range tables { + _, _ = mcmp.ExecAndIgnore("delete from " + table) + } + } + + deleteAll() + + // disable only_full_group_by + mcmp.Exec("set sql_mode=''") + + // insert data + mcmp.Exec(empInsert) + mcmp.Exec(deptInsert) + + return mcmp, func() { + deleteAll() + mcmp.Close() + cluster.PanicHandler(t) + } +} From d090b670e92a82661e944d4a0f51e55a457f1e43 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 29 Nov 2023 08:20:19 +0100 Subject: [PATCH 4/5] test: introduce the new query gen Signed-off-by: Andres Taylor --- .../queries/query_fuzzing/fuzzing_test.go | 67 ++++++++++--------- .../vtgate/queries/query_fuzzing/query_gen.go | 11 +-- .../queries/query_fuzzing/query_gen2.go | 36 ++++++++++ 3 files changed, 77 insertions(+), 37 deletions(-) create mode 100644 go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go index 6b6f9a06e0e..13aa7f98c91 100644 --- a/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go @@ -18,8 +18,8 @@ package query_fuzzing import ( _ "embed" + "errors" "fmt" - "math/rand" "strings" "testing" "time" @@ -55,8 +55,6 @@ func helperTest(t *testing.T, query string) { } func TestFuzzQueries(t *testing.T) { - t.Skip("Skip CI; random expressions generate too many failures to properly limit") - mcmp, closer := start(t) defer closer() @@ -90,39 +88,42 @@ func TestFuzzQueries(t *testing.T) { // continue testing after an error if and only if testFailingQueries is true for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) { seed := time.Now().UnixNano() - genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false) - qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables) - qg.randomQuery() - query := sqlparser.String(qg.stmt) - _, vtErr := mcmp.ExecAllowAndCompareError(query) - - // this assumes all queries are valid mysql queries - if vtErr != nil { - fmt.Printf("seed: %d\n", seed) - fmt.Println(query) - fmt.Println(vtErr) - - if stopOnMustFixError { - // results mismatched - if strings.Contains(vtErr.Error(), "results mismatched") { - simplified := simplifyResultsMismatchedQuery(t, query) - fmt.Printf("final simplified query: %s\n", simplified) - break - } - // EOF - if sqlError, ok := vtErr.(*sqlerror.SQLError); ok && strings.Contains(sqlError.Message, "EOF") { - break + // genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false) + // qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables) + qg := &maruts{} + query := sqlparser.String(qg.randomQuery()) + t.Run(query, func(t *testing.T) { + _, vtErr := mcmp.ExecAllowAndCompareError(query) + + // this assumes all queries are valid mysql queries + if vtErr != nil { + fmt.Printf("seed: %d\n", seed) + fmt.Println(query) + fmt.Println(vtErr) + + if stopOnMustFixError { + // results mismatched + if strings.Contains(vtErr.Error(), "results mismatched") { + simplified := simplifyResultsMismatchedQuery(t, query) + fmt.Printf("final simplified query: %s\n", simplified) + t.Fatal("failed") + } + // EOF + var sqlError *sqlerror.SQLError + if errors.As(vtErr, &sqlError) && strings.Contains(sqlError.Message, "EOF") { + t.Fatal("failed") + } } - } - // restart the mysql and vitess connections in case something bad happened - closer() - mcmp, closer = start(t) + // restart the mysql and vitess connections in case something bad happened + closer() + mcmp, closer = start(t) - fmt.Printf("\n\n\n") - queryFailCount++ - } - queryCount++ + fmt.Printf("\n\n\n") + queryFailCount++ + } + queryCount++ + }) } fmt.Printf("Queries successfully executed: %d\n", queryCount) fmt.Printf("Queries failed: %d\n", queryFailCount) diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go index 8b665031364..4a66ac87feb 100644 --- a/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen.go @@ -44,20 +44,17 @@ type ( schemaTables []tableT sel *sqlparser.Select } - // queryGenerator generates queries, which can either be unions or select statements queryGenerator struct { stmt sqlparser.SelectStatement selGen *selectGenerator } - column struct { name string // TODO: perhaps remove tableName and always pass columns through a tableT tableName string typ string } - tableT struct { // the tableT struct can be used to represent the schema of a table or a derived table // in the former case tableExpr will be a sqlparser.TableName, in the latter a sqlparser.DerivedTable @@ -67,6 +64,9 @@ type ( alias string cols []column } + queryGen interface { + randomQuery() sqlparser.SelectStatement + } ) var _ sqlparser.ExprGenerator = (*tableT)(nil) @@ -74,6 +74,8 @@ var _ sqlparser.ExprGenerator = (*column)(nil) var _ sqlparser.QueryGenerator = (*selectGenerator)(nil) var _ sqlparser.QueryGenerator = (*queryGenerator)(nil) +var _ queryGen = (*queryGenerator)(nil) + func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, maxTables, maxAggrs, maxGBs int, schemaTables []tableT) *queryGenerator { return &queryGenerator{ selGen: newSelectGenerator(r, genConfig, maxTables, maxAggrs, maxGBs, schemaTables), @@ -196,13 +198,14 @@ func (qg *queryGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGenerat func (sg *selectGenerator) IsQueryGenerator() {} func (qg *queryGenerator) IsQueryGenerator() {} -func (qg *queryGenerator) randomQuery() { +func (qg *queryGenerator) randomQuery() sqlparser.SelectStatement { if qg.selGen.r.Intn(10) < 1 && testFailingQueries { qg.createUnion() } else { qg.selGen.randomSelect() qg.stmt = qg.selGen.sel } + return qg.stmt } // createUnion creates a simple UNION or UNION ALL; no LIMIT or ORDER BY diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go new file mode 100644 index 00000000000..23158bee35d --- /dev/null +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package query_fuzzing + +import ( + "vitess.io/vitess/go/vt/sqlparser" +) + +// maruts is a query generator that generates random queries to be used in fuzzing +type maruts struct { +} + +func (m *maruts) randomQuery() sqlparser.SelectStatement { + expr := &sqlparser.AliasedExpr{Expr: sqlparser.NewIntLiteral("1")} + tableExpr := &sqlparser.AliasedTableExpr{Expr: sqlparser.NewTableName("dual")} + return &sqlparser.Select{ + SelectExprs: sqlparser.SelectExprs{expr}, + From: []sqlparser.TableExpr{tableExpr}, + } +} + +var _ queryGen = (*maruts)(nil) From 5226e8cb896a4a6b6b657e644cad11a988e01e8f Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 29 Nov 2023 14:24:56 +0100 Subject: [PATCH 5/5] wip --- .../queries/query_fuzzing/query_gen2.go | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go index 23158bee35d..4b481f00da5 100644 --- a/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go +++ b/go/test/endtoend/vtgate/queries/query_fuzzing/query_gen2.go @@ -17,18 +17,35 @@ limitations under the License. package query_fuzzing import ( + "math/rand" + "vitess.io/vitess/go/vt/sqlparser" ) -// maruts is a query generator that generates random queries to be used in fuzzing -type maruts struct { -} +type ( + // maruts is a query generator that generates random queries to be used in fuzzing + maruts struct { + cfg config + } + + config struct { + r *rand.Rand + genConfig sqlparser.ExprGeneratorConfig + maxTables int + maxAggrs int + maxGBs int + schemaTables []tableT + } +) func (m *maruts) randomQuery() sqlparser.SelectStatement { - expr := &sqlparser.AliasedExpr{Expr: sqlparser.NewIntLiteral("1")} + exprGen := sqlparser.NewGenerator(m.cfg.r, 2) + expr := exprGen.Expression(m.cfg.genConfig) + + ae := &sqlparser.AliasedExpr{Expr: expr} tableExpr := &sqlparser.AliasedTableExpr{Expr: sqlparser.NewTableName("dual")} return &sqlparser.Select{ - SelectExprs: sqlparser.SelectExprs{expr}, + SelectExprs: sqlparser.SelectExprs{ae}, From: []sqlparser.TableExpr{tableExpr}, } }