Skip to content

Commit

Permalink
Foreign key on update action with non literal values (#14278)
Browse files Browse the repository at this point in the history 
Signed-off-by: Harshit Gangal <[email protected]>
Signed-off-by: Manan Gupta <[email protected]>
Co-authored-by: Manan Gupta <[email protected]>
  • Loading branch information
@harshit-gangal @GuptaManan100
harshit-gangal and GuptaManan100 authored Nov 10, 2023
1 parent 23bca17 commit e61eae0
Show file tree
Hide file tree
Showing 23 changed files with 2,333 additions and 877 deletions.
170 changes: 46 additions & 124 deletions go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go
View file

Large diffs are not rendered by default.

176 changes: 176 additions & 0 deletions go/test/endtoend/vtgate/foreignkey/fk_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

"vitess.io/vitess/go/test/endtoend/cluster"
"vitess.io/vitess/go/test/endtoend/utils"
"vitess.io/vitess/go/vt/log"
binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
topodatapb "vitess.io/vitess/go/vt/proto/topodata"
"vitess.io/vitess/go/vt/vtgate/vtgateconn"
Expand Down Expand Up @@ -775,6 +776,181 @@ func TestFkScenarios(t *testing.T) {
}
}

// TestFkQueries is for testing a specific set of queries one after the other.
func TestFkQueries(t *testing.T) {
// Wait for schema-tracking to be complete.
waitForSchemaTrackingForFkTables(t)
// Remove all the foreign key constraints for all the replicas.
// We can then verify that the replica, and the primary have the same data, to ensure
// that none of the queries ever lead to cascades/updates on MySQL level.
for _, ks := range []string{shardedKs, unshardedKs} {
replicas := getReplicaTablets(ks)
for _, replica := range replicas {
removeAllForeignKeyConstraints(t, replica, ks)
}
}

testcases := []struct {
name string
queries []string
}{
{
name: "Non-literal update",
queries: []string{
"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"update fk_t10 set col = id + 3",
},
}, {
name: "Non-literal update with order by",
queries: []string{
"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"update fk_t10 set col = id + 3 order by id desc",
},
}, {
name: "Non-literal update with order by that require parent and child foreign keys verification - success",
queries: []string{
"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8)",
"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t13 (id, col) values (1,1),(2,2)",
"update fk_t11 set col = id + 3 where id >= 3",
},
}, {
name: "Non-literal update with order by that require parent and child foreign keys verification - parent fails",
queries: []string{
"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"update fk_t11 set col = id + 3",
},
}, {
name: "Non-literal update with order by that require parent and child foreign keys verification - child fails",
queries: []string{
"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8)",
"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t13 (id, col) values (1,1),(2,2)",
"update fk_t11 set col = id + 3",
},
}, {
name: "Single column update in a multi-col table - success",
queries: []string{
"insert into fk_multicol_t1 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
"insert into fk_multicol_t2 (id, cola, colb) values (1, 1, 1)",
"update fk_multicol_t1 set colb = 4 + (colb) where id = 2",
},
}, {
name: "Single column update in a multi-col table - restrict failure",
queries: []string{
"insert into fk_multicol_t1 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
"insert into fk_multicol_t2 (id, cola, colb) values (1, 1, 1)",
"update fk_multicol_t1 set colb = 4 + (colb) where id = 1",
},
}, {
name: "Single column update in multi-col table - cascade and set null",
queries: []string{
"insert into fk_multicol_t15 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
"insert into fk_multicol_t16 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
"insert into fk_multicol_t17 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
"update fk_multicol_t15 set colb = 4 + (colb) where id = 1",
},
}, {
name: "Non literal update that evaluates to NULL - restricted",
queries: []string{
"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t13 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"update fk_t10 set col = id + null where id = 1",
},
}, {
name: "Non literal update that evaluates to NULL - success",
queries: []string{
"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
"update fk_t10 set col = id + null where id = 1",
},
}, {
name: "Multi column foreign key update with one literal and one non-literal update",
queries: []string{
"insert into fk_multicol_t15 (id, cola, colb) values (1,1,1),(2,2,2)",
"insert into fk_multicol_t16 (id, cola, colb) values (1,1,1),(2,2,2)",
"update fk_multicol_t15 set cola = 3, colb = (id * 2) - 2",
},
},
}

for _, testcase := range testcases {
t.Run(testcase.name, func(t *testing.T) {
mcmp, closer := start(t)
defer closer()
_ = utils.Exec(t, mcmp.VtConn, "use `uks`")

// Ensure that the Vitess database is originally empty
ensureDatabaseState(t, mcmp.VtConn, true)
ensureDatabaseState(t, mcmp.MySQLConn, true)

for _, query := range testcase.queries {
_, _ = mcmp.ExecAllowAndCompareError(query)
if t.Failed() {
break
}
}

// ensure Vitess database has some data. This ensures not all the commands failed.
ensureDatabaseState(t, mcmp.VtConn, false)
// Verify the consistency of the data.
verifyDataIsCorrect(t, mcmp, 1)
})
}
}

// TestFkOneCase is for testing a specific set of queries. On the CI this test won't run since we'll keep the queries empty.
func TestFkOneCase(t *testing.T) {
queries := []string{}
if len(queries) == 0 {
t.Skip("No queries to test")
}
// Wait for schema-tracking to be complete.
waitForSchemaTrackingForFkTables(t)
// Remove all the foreign key constraints for all the replicas.
// We can then verify that the replica, and the primary have the same data, to ensure
// that none of the queries ever lead to cascades/updates on MySQL level.
for _, ks := range []string{shardedKs, unshardedKs} {
replicas := getReplicaTablets(ks)
for _, replica := range replicas {
removeAllForeignKeyConstraints(t, replica, ks)
}
}

mcmp, closer := start(t)
defer closer()
_ = utils.Exec(t, mcmp.VtConn, "use `uks`")

// Ensure that the Vitess database is originally empty
ensureDatabaseState(t, mcmp.VtConn, true)
ensureDatabaseState(t, mcmp.MySQLConn, true)

for _, query := range queries {
_, _ = mcmp.ExecAllowAndCompareError(query)
if t.Failed() {
log.Errorf("Query failed - %v", query)
break
}
}
vitessData := collectFkTablesState(mcmp.VtConn)
for idx, table := range fkTables {
log.Errorf("Vitess data for %v -\n%v", table, vitessData[idx].Rows)
}

// ensure Vitess database has some data. This ensures not all the commands failed.
ensureDatabaseState(t, mcmp.VtConn, false)
// Verify the consistency of the data.
verifyDataIsCorrect(t, mcmp, 1)
}

func TestCyclicFks(t *testing.T) {
mcmp, closer := start(t)
defer closer()
Expand Down
114 changes: 114 additions & 0 deletions go/test/endtoend/vtgate/foreignkey/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,21 @@ package foreignkey
import (
"database/sql"
"fmt"
"math/rand"
"strings"
"testing"
"time"

"github.com/stretchr/testify/require"

"vitess.io/vitess/go/mysql"
"vitess.io/vitess/go/sqltypes"
"vitess.io/vitess/go/test/endtoend/cluster"
"vitess.io/vitess/go/test/endtoend/utils"
)

var supportedOpps = []string{"*", "+", "-"}

// getTestName prepends whether the test is for a sharded keyspace or not to the test name.
func getTestName(testName string, testSharded bool) string {
if testSharded {
Expand All @@ -41,6 +47,32 @@ func isMultiColFkTable(tableName string) bool {
return strings.Contains(tableName, "multicol")
}

func (fz *fuzzer) generateExpression(length int, cols ...string) string {
expr := fz.getColOrInt(cols...)
if length == 1 {
return expr
}
rhsExpr := fz.generateExpression(length-1, cols...)
op := supportedOpps[rand.Intn(len(supportedOpps))]
return fmt.Sprintf("%v %s (%v)", expr, op, rhsExpr)
}

// getColOrInt gets a column or an integer/NULL literal with equal probability.
func (fz *fuzzer) getColOrInt(cols ...string) string {
if len(cols) == 0 || rand.Intn(2) == 0 {
return convertIntValueToString(rand.Intn(1 + fz.maxValForCol))
}
return cols[rand.Intn(len(cols))]
}

// convertIntValueToString converts the given value to a string
func convertIntValueToString(value int) string {
if value == 0 {
return "NULL"
}
return fmt.Sprintf("%d", value)
}

// waitForSchemaTrackingForFkTables waits for schema tracking to have run and seen the tables used
// for foreign key tests.
func waitForSchemaTrackingForFkTables(t *testing.T) {
Expand Down Expand Up @@ -142,3 +174,85 @@ func compareVitessAndMySQLErrors(t *testing.T, vtErr, mysqlErr error) {
out := fmt.Sprintf("Vitess and MySQL are not erroring the same way.\nVitess error: %v\nMySQL error: %v", vtErr, mysqlErr)
t.Error(out)
}

// ensureDatabaseState ensures that the database is either empty or not.
func ensureDatabaseState(t *testing.T, vtconn *mysql.Conn, empty bool) {
results := collectFkTablesState(vtconn)
isEmpty := true
for _, res := range results {
if len(res.Rows) > 0 {
isEmpty = false
}
}
require.Equal(t, isEmpty, empty)
}

// verifyDataIsCorrect verifies that the data in MySQL database matches the data in the Vitess database.
func verifyDataIsCorrect(t *testing.T, mcmp utils.MySQLCompare, concurrency int) {
// For single concurrent thread, we run all the queries on both MySQL and Vitess, so we can verify correctness
// by just checking if the data in MySQL and Vitess match.
if concurrency == 1 {
for _, table := range fkTables {
query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table)
mcmp.Exec(query)
}
} else {
// For higher concurrency, we don't have MySQL data to verify everything is fine,
// so we'll have to do something different.
// We run LEFT JOIN queries on all the parent and child tables linked by foreign keys
// to make sure that nothing is broken in the database.
for _, reference := range fkReferences {
query := fmt.Sprintf("select %v.id from %v left join %v on (%v.col = %v.col) where %v.col is null and %v.col is not null", reference.childTable, reference.childTable, reference.parentTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable)
if isMultiColFkTable(reference.childTable) {
query = fmt.Sprintf("select %v.id from %v left join %v on (%v.cola = %v.cola and %v.colb = %v.colb) where %v.cola is null and %v.cola is not null and %v.colb is not null", reference.childTable, reference.childTable, reference.parentTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable, reference.childTable)
}
res, err := mcmp.VtConn.ExecuteFetch(query, 1000, false)
require.NoError(t, err)
require.Zerof(t, len(res.Rows), "Query %v gave non-empty results", query)
}
}
// We also verify that the results in Primary and Replica table match as is.
for _, keyspace := range clusterInstance.Keyspaces {
for _, shard := range keyspace.Shards {
var primaryTab, replicaTab *cluster.Vttablet
for _, vttablet := range shard.Vttablets {
if vttablet.Type == "primary" {
primaryTab = vttablet
} else {
replicaTab = vttablet
}
}
require.NotNil(t, primaryTab)
require.NotNil(t, replicaTab)
checkReplicationHealthy(t, replicaTab)
cluster.WaitForReplicationPos(t, primaryTab, replicaTab, true, 1*time.Minute)
primaryConn, err := utils.GetMySQLConn(primaryTab, fmt.Sprintf("vt_%v", keyspace.Name))
require.NoError(t, err)
replicaConn, err := utils.GetMySQLConn(replicaTab, fmt.Sprintf("vt_%v", keyspace.Name))
require.NoError(t, err)
primaryRes := collectFkTablesState(primaryConn)
replicaRes := collectFkTablesState(replicaConn)
verifyDataMatches(t, primaryRes, replicaRes)
}
}
}

// verifyDataMatches verifies that the two list of results are the same.
func verifyDataMatches(t *testing.T, resOne []*sqltypes.Result, resTwo []*sqltypes.Result) {
require.EqualValues(t, len(resTwo), len(resOne), "Res 1 - %v, Res 2 - %v", resOne, resTwo)
for idx, resultOne := range resOne {
resultTwo := resTwo[idx]
require.True(t, resultOne.Equal(resultTwo), "Data for %v doesn't match\nRows 1\n%v\nRows 2\n%v", fkTables[idx], resultOne.Rows, resultTwo.Rows)
}
}

// collectFkTablesState collects the data stored in the foreign key tables for the given connection.
func collectFkTablesState(conn *mysql.Conn) []*sqltypes.Result {
var tablesData []*sqltypes.Result
for _, table := range fkTables {
query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table)
res, _ := conn.ExecuteFetch(query, 10000, true)
tablesData = append(tablesData, res)
}
return tablesData
}
21 changes: 20 additions & 1 deletion go/vt/vtgate/engine/cached_size.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit e61eae0

Please sign in to comment.