query fuzzing: test cleaning and renaming

go/test/endtoend/vtgate/queries/aggregation/main_test.go

@@ -23,10 +23,9 @@ import (
 	"os"
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 var (

go/test/endtoend/vtgate/queries/derived/main_test.go

@@ -23,10 +23,9 @@ import (
 	"os"
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 var (

go/test/endtoend/vtgate/queries/foundrows/main_test.go

@@ -23,10 +23,9 @@ import (
 	"os"
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 var (

go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go

@@ -21,14 +21,12 @@ import (
 	"fmt"
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"github.com/stretchr/testify/assert"
-
 	"github.com/stretchr/testify/require"
 
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 func start(t *testing.T) (utils.MySQLCompare, func()) {

go/test/endtoend/vtgate/queries/informationschema/main_test.go

@@ -23,10 +23,9 @@ import (
 	"os"
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 var (

go/test/endtoend/vtgate/queries/lookup_queries/main_test.go

@@ -25,12 +25,10 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
-	"vitess.io/vitess/go/vt/vtgate/planbuilder"
-
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
+	"vitess.io/vitess/go/vt/vtgate/planbuilder"
 )
 
 var (

go/test/endtoend/vtgate/queries/misc/main_test.go

@@ -23,10 +23,9 @@ import (
 	"os"
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 var (

go/test/endtoend/vtgate/queries/normalize/normalize_test.go

@@ -28,9 +28,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"vitess.io/vitess/go/mysql"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 func TestNormalizeAllFields(t *testing.T) {

go/test/endtoend/vtgate/queries/orderby/main_test.go

@@ -23,10 +23,9 @@ import (
 	"os"
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 var (

go/test/endtoend/vtgate/queries/orderby/orderby_test.go

@@ -19,11 +19,10 @@ package orderby
 import (
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"github.com/stretchr/testify/require"
 
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 func start(t *testing.T) (utils.MySQLCompare, func()) {

go/test/endtoend/vtgate/queries/orderby/without_schematracker/main_test.go

@@ -23,10 +23,9 @@ import (
 	"os"
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 var (

go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go

@@ -19,11 +19,10 @@ package orderby
 import (
 	"testing"
 
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"github.com/stretchr/testify/require"
 
 	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 func start(t *testing.T) (utils.MySQLCompare, func()) {

go/test/endtoend/vtgate/queries/query_fuzzing/dept_insert.sql

@@ -0,0 +1,5 @@
+INSERT INTO dept(deptno, dname, loc)
+VALUES ('10', 'ACCOUNTING', 'NEW YORK'),
+       ('20', 'RESEARCH', 'DALLAS'),
+       ('30', 'SALES', 'CHICAGO'),
+       ('40', 'OPERATIONS', 'BOSTON')

go/test/endtoend/vtgate/queries/query_fuzzing/emp_insert.sql

@@ -0,0 +1,15 @@
+INSERT INTO emp(empno, ename, job, mgr, hiredate, sal, comm, deptno)
+VALUES (7369, 'SMITH', 'CLERK', 7902, '1980-12-17', 800, NULL, 20),
+       (7499, 'ALLEN', 'SALESMAN', 7698, '1981-02-20', 1600, 300, 30),
+       (7521, 'WARD', 'SALESMAN', 7698, '1981-02-22', 1250, 500, 30),
+       (7566, 'JONES', 'MANAGER', 7839, '1981-04-02', 2975, NULL, 20),
+       (7654, 'MARTIN', 'SALESMAN', 7698, '1981-09-28', 1250, 1400, 30),
+       (7698, 'BLAKE', 'MANAGER', 7839, '1981-05-01', 2850, NULL, 30),
+       (7782, 'CLARK', 'MANAGER', 7839, '1981-06-09', 2450, NULL, 10),
+       (7788, 'SCOTT', 'ANALYST', 7566, '1982-12-09', 3000, NULL, 20),
+       (7839, 'KING', 'PRESIDENT', NULL, '1981-11-17', 5000, NULL, 10),
+       (7844, 'TURNER', 'SALESMAN', 7698, '1981-09-08', 1500, 0, 30),
+       (7876, 'ADAMS', 'CLERK', 7788, '1983-01-12', 1100, NULL, 20),
+       (7900, 'JAMES', 'CLERK', 7698, '1981-12-03', 950, NULL, 30),
+       (7902, 'FORD', 'ANALYST', 7566, '1981-12-03', 3000, NULL, 20),
+       (7934, 'MILLER', 'CLERK', 7782, '1982-01-23', 1300, NULL, 10)

go/test/endtoend/vtgate/queries/query_fuzzing/fuzzing_test.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package query_fuzzing
+
+import (
+	_ "embed"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/mysql/sqlerror"
+	"vitess.io/vitess/go/test/endtoend/utils"
+	"vitess.io/vitess/go/vt/sqlparser"
+)
+
+// this test uses the AST defined in the sqlparser package to randomly generate queries
+
+// if true then execution will always stop on a "must fix" error: a results mismatched or EOF
+const stopOnMustFixError = false
+
+//go:embed emp_insert.sql
+var empInsert string
+
+//go:embed dept_insert.sql
+var deptInsert string
+
+func helperTest(t *testing.T, query string) {
+	t.Helper()
+	t.Run(query, func(t *testing.T) {
+		mcmp, closer := start(t)
+		defer closer()
+
+		result, err := mcmp.ExecAllowAndCompareError(query)
+		fmt.Println(result)
+		fmt.Println(err)
+	})
+}
+
+func TestFuzzQueries(t *testing.T) {
+	mcmp, closer := start(t)
+	defer closer()
+
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
+
+	// specify the schema (that is defined in schema.sql)
+	schemaTables := []tableT{
+		{tableExpr: sqlparser.NewTableName("emp")},
+		{tableExpr: sqlparser.NewTableName("dept")},
+	}
+	schemaTables[0].addColumns([]column{
+		{name: "empno", typ: "bigint"},
+		{name: "ename", typ: "varchar"},
+		{name: "job", typ: "varchar"},
+		{name: "mgr", typ: "bigint"},
+		{name: "hiredate", typ: "date"},
+		{name: "sal", typ: "bigint"},
+		{name: "comm", typ: "bigint"},
+		{name: "deptno", typ: "bigint"},
+	}...)
+	schemaTables[1].addColumns([]column{
+		{name: "deptno", typ: "bigint"},
+		{name: "dname", typ: "varchar"},
+		{name: "loc", typ: "varchar"},
+	}...)
+
+	endBy := time.Now().Add(1 * time.Second)
+
+	var queryCount, queryFailCount int
+	// continue testing after an error if and only if testFailingQueries is true
+	for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) {
+		seed := time.Now().UnixNano()
+		// genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false)
+		// qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
+		qg := &maruts{}
+		query := sqlparser.String(qg.randomQuery())
+		t.Run(query, func(t *testing.T) {
+			_, vtErr := mcmp.ExecAllowAndCompareError(query)
+
+			// this assumes all queries are valid mysql queries
+			if vtErr != nil {
+				fmt.Printf("seed: %d\n", seed)
+				fmt.Println(query)
+				fmt.Println(vtErr)
+
+				if stopOnMustFixError {
+					// results mismatched
+					if strings.Contains(vtErr.Error(), "results mismatched") {
+						simplified := simplifyResultsMismatchedQuery(t, query)
+						fmt.Printf("final simplified query: %s\n", simplified)
+						t.Fatal("failed")
+					}
+					// EOF
+					var sqlError *sqlerror.SQLError
+					if errors.As(vtErr, &sqlError) && strings.Contains(sqlError.Message, "EOF") {
+						t.Fatal("failed")
+					}
+				}
+
+				// restart the mysql and vitess connections in case something bad happened
+				closer()
+				mcmp, closer = start(t)
+
+				fmt.Printf("\n\n\n")
+				queryFailCount++
+			}
+			queryCount++
+		})
+	}
+	fmt.Printf("Queries successfully executed: %d\n", queryCount)
+	fmt.Printf("Queries failed: %d\n", queryFailCount)
+}