diff --git a/go/test/endtoend/vtgate/foreignkey/fk_test.go b/go/test/endtoend/vtgate/foreignkey/fk_test.go new file mode 100644 index 00000000000..d61aea0aed9 --- /dev/null +++ b/go/test/endtoend/vtgate/foreignkey/fk_test.go @@ -0,0 +1,51 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foreignkey + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/endtoend/utils" +) + +// TestInsertions tests that insertions work as expected when foreign key management is enabled in Vitess. +func TestInsertions(t *testing.T) { + conn, closer := start(t) + defer closer() + + // insert some data. + utils.Exec(t, conn, `insert into t1(id, col) values (100, 123),(10, 12),(1, 13),(1000, 1234)`) + + // Verify that inserting data into a table that has shard scoped foreign keys works. + utils.Exec(t, conn, `insert into t2(id, col) values (100, 125), (1, 132)`) + // Verify that insertion fails if the data doesn't follow the fk constraint. + _, err := utils.ExecAllowError(t, conn, `insert into t2(id, col) values (1310, 125)`) + require.ErrorContains(t, err, "Cannot add or update a child row: a foreign key constraint fails") + // Verify that insertion fails if the table has cross-shard foreign keys (even if the data follows the constraints). + _, err = utils.ExecAllowError(t, conn, `insert into t3(id, col) values (100, 100)`) + require.ErrorContains(t, err, "VT12002: unsupported: cross-shard foreign keys") + + // insert some data in a table with multicol vindex. + utils.Exec(t, conn, `insert into multicol_tbl1(cola, colb, colc, msg) values (100, 'a', 'b', 'msg'), (101, 'c', 'd', 'msg2')`) + // Verify that inserting data into a table that has shard scoped multi-column foreign keys works. + utils.Exec(t, conn, `insert into multicol_tbl2(cola, colb, colc, msg) values (100, 'a', 'b', 'msg3')`) + // Verify that insertion fails if the data doesn't follow the fk constraint. + _, err = utils.ExecAllowError(t, conn, `insert into multicol_tbl2(cola, colb, colc, msg) values (103, 'c', 'd', 'msg2')`) + require.ErrorContains(t, err, "Cannot add or update a child row: a foreign key constraint fails") +} diff --git a/go/test/endtoend/vtgate/foreignkey/main_test.go b/go/test/endtoend/vtgate/foreignkey/main_test.go new file mode 100644 index 00000000000..eaeae4eeb9b --- /dev/null +++ b/go/test/endtoend/vtgate/foreignkey/main_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foreignkey + +import ( + "context" + _ "embed" + "flag" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/endtoend/utils" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + shardedKs = "ks" + Cell = "test" + + //go:embed sharded_schema.sql + shardedSchemaSQL string + + //go:embed sharded_vschema.json + shardedVSchema string +) + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(Cell, "localhost") + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + // Start keyspace + sKs := &cluster.Keyspace{ + Name: shardedKs, + SchemaSQL: shardedSchemaSQL, + VSchema: shardedVSchema, + } + + err = clusterInstance.StartKeyspace(*sKs, []string{"-80", "80-"}, 0, false) + if err != nil { + return 1 + } + + // Start vtgate + err = clusterInstance.StartVtgate() + if err != nil { + return 1 + } + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + + return m.Run() + }() + os.Exit(exitCode) +} + +func start(t *testing.T) (*mysql.Conn, func()) { + conn, err := mysql.Connect(context.Background(), &vtParams) + require.NoError(t, err) + + deleteAll := func() { + tables := []string{"t3", "t2", "t1", "multicol_tbl2", "multicol_tbl1"} + for _, table := range tables { + _ = utils.Exec(t, conn, "delete from "+table) + } + } + + deleteAll() + + return conn, func() { + deleteAll() + conn.Close() + cluster.PanicHandler(t) + } +} diff --git a/go/test/endtoend/vtgate/foreignkey/sharded_schema.sql b/go/test/endtoend/vtgate/foreignkey/sharded_schema.sql new file mode 100644 index 00000000000..47432e0998b --- /dev/null +++ b/go/test/endtoend/vtgate/foreignkey/sharded_schema.sql @@ -0,0 +1,41 @@ +create table t1 +( + id bigint, + col bigint, + primary key (id) +) Engine = InnoDB; + +create table t2 +( + id bigint, + col bigint, + primary key (id), + foreign key (id) references t1 (id) +) Engine = InnoDB; + +create table t3 +( + id bigint, + col bigint, + primary key (id), + foreign key (col) references t1 (id) +) Engine = InnoDB; + +create table multicol_tbl1 +( + cola bigint, + colb varbinary(50), + colc varchar(50), + msg varchar(50), + primary key (cola, colb, colc) +) Engine = InnoDB; + +create table multicol_tbl2 +( + cola bigint, + colb varbinary(50), + colc varchar(50), + msg varchar(50), + primary key (cola, colb, colc), + foreign key (cola, colb, colc) references multicol_tbl1 (cola, colb, colc) +) Engine = InnoDB; diff --git a/go/test/endtoend/vtgate/foreignkey/sharded_vschema.json b/go/test/endtoend/vtgate/foreignkey/sharded_vschema.json new file mode 100644 index 00000000000..e55185f25b0 --- /dev/null +++ b/go/test/endtoend/vtgate/foreignkey/sharded_vschema.json @@ -0,0 +1,67 @@ +{ + "sharded": true, + "foreignKeyMode": "FK_MANAGED", + "vindexes": { + "xxhash": { + "type": "xxhash" + }, + "multicol_vdx": { + "type": "multicol", + "params": { + "column_count": "3", + "column_bytes": "1,3,4", + "column_vindex": "hash,binary,unicode_loose_xxhash" + } + } + }, + "tables": { + "t1": { + "column_vindexes": [ + { + "column": "id", + "name": "xxhash" + } + ] + }, + "t2": { + "column_vindexes": [ + { + "column": "id", + "name": "xxhash" + } + ] + }, + "t3": { + "column_vindexes": [ + { + "column": "id", + "name": "xxhash" + } + ] + }, + "multicol_tbl1": { + "column_vindexes": [ + { + "columns": [ + "cola", + "colb", + "colc" + ], + "name": "multicol_vdx" + } + ] + }, + "multicol_tbl2": { + "column_vindexes": [ + { + "columns": [ + "cola", + "colb", + "colc" + ], + "name": "multicol_vdx" + } + ] + } + } +} \ No newline at end of file diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index 5bcc6c2e7ac..68845ac73fa 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -2472,3 +2472,33 @@ func (ty KillType) ToString() string { return ConnectionStr } } + +// Indexes returns true, if the list of columns contains all the elements in the other list. +// It also returns the indexes of the columns in the list. +func (cols Columns) Indexes(subSetCols Columns) (bool, []int) { + var indexes []int + for _, subSetCol := range subSetCols { + colFound := false + for idx, col := range cols { + if col.Equal(subSetCol) { + colFound = true + indexes = append(indexes, idx) + break + } + } + if !colFound { + return false, nil + } + } + return true, indexes +} + +// MakeColumns is used to make a list of columns from a list of strings. +// This function is meant to be used in testing code. +func MakeColumns(colNames ...string) Columns { + var cols Columns + for _, name := range colNames { + cols = append(cols, NewIdentifierCI(name)) + } + return cols +} diff --git a/go/vt/sqlparser/ast_funcs_test.go b/go/vt/sqlparser/ast_funcs_test.go index b6a79da45ab..7bec47df96f 100644 --- a/go/vt/sqlparser/ast_funcs_test.go +++ b/go/vt/sqlparser/ast_funcs_test.go @@ -134,3 +134,41 @@ func TestSQLTypeToQueryType(t *testing.T) { }) } } + +// TestColumns_Indexes verifies the functionality of Indexes method on Columns. +func TestColumns_Indexes(t *testing.T) { + tests := []struct { + name string + cols Columns + subSetCols Columns + indexesWanted []int + }{ + { + name: "Not a subset", + cols: MakeColumns("col1", "col2", "col3"), + subSetCols: MakeColumns("col2", "col4"), + }, { + name: "Subset with 1 value", + cols: MakeColumns("col1", "col2", "col3"), + subSetCols: MakeColumns("col2"), + indexesWanted: []int{1}, + }, { + name: "Subset with multiple values", + cols: MakeColumns("col1", "col2", "col3", "col4", "col5"), + subSetCols: MakeColumns("col3", "col5", "col1"), + indexesWanted: []int{2, 4, 0}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + isSubset, indexes := tt.cols.Indexes(tt.subSetCols) + if tt.indexesWanted == nil { + require.False(t, isSubset) + require.Nil(t, indexes) + return + } + require.True(t, isSubset) + require.EqualValues(t, tt.indexesWanted, indexes) + }) + } +} diff --git a/go/vt/sqlparser/normalizer_test.go b/go/vt/sqlparser/normalizer_test.go index 8e40dfe9f1a..2b0a4b52122 100644 --- a/go/vt/sqlparser/normalizer_test.go +++ b/go/vt/sqlparser/normalizer_test.go @@ -436,6 +436,34 @@ func TestNormalizeValidSQL(t *testing.T) { } } +func TestNormalizeOneCasae(t *testing.T) { + testOne := struct { + input, output string + }{ + input: "", + output: "", + } + if testOne.input == "" { + t.Skip("empty test case") + } + tree, err := Parse(testOne.input) + require.NoError(t, err, testOne.input) + // Skip the test for the queries that do not run the normalizer + if !CanNormalize(tree) { + return + } + bv := make(map[string]*querypb.BindVariable) + known := make(BindVars) + err = Normalize(tree, NewReservedVars("vtg", known), bv) + require.NoError(t, err) + normalizerOutput := String(tree) + if normalizerOutput == "otheradmin" || normalizerOutput == "otherread" { + return + } + _, err = Parse(normalizerOutput) + require.NoError(t, err, normalizerOutput) +} + func TestGetBindVars(t *testing.T) { stmt, err := Parse("select * from t where :v1 = :v2 and :v2 = :v3 and :v4 in ::v5") if err != nil { diff --git a/go/vt/vterrors/code.go b/go/vt/vterrors/code.go index 2afccb561e2..9f178cc9a13 100644 --- a/go/vt/vterrors/code.go +++ b/go/vt/vterrors/code.go @@ -80,6 +80,7 @@ var ( VT10001 = errorWithoutState("VT10001", vtrpcpb.Code_ABORTED, "foreign key constraints are not allowed", "Foreign key constraints are not allowed, see https://vitess.io/blog/2021-06-15-online-ddl-why-no-fk/.") VT12001 = errorWithoutState("VT12001", vtrpcpb.Code_UNIMPLEMENTED, "unsupported: %s", "This statement is unsupported by Vitess. Please rewrite your query to use supported syntax.") + VT12002 = errorWithoutState("VT12002", vtrpcpb.Code_UNIMPLEMENTED, "unsupported: cross-shard foreign keys", "Vitess does not support cross shard foreign keys.") // VT13001 General Error VT13001 = errorWithoutState("VT13001", vtrpcpb.Code_INTERNAL, "[BUG] %s", "This error should not happen and is a bug. Please file an issue on GitHub: https://github.com/vitessio/vitess/issues/new/choose.") @@ -143,6 +144,7 @@ var ( VT09015, VT10001, VT12001, + VT12002, VT13001, VT13002, VT14001, diff --git a/go/vt/vtgate/planbuilder/ddl.go b/go/vt/vtgate/planbuilder/ddl.go index 871b8adc38a..1557a1d1d8f 100644 --- a/go/vt/vtgate/planbuilder/ddl.go +++ b/go/vt/vtgate/planbuilder/ddl.go @@ -5,6 +5,7 @@ import ( "fmt" "vitess.io/vitess/go/vt/key" + vschemapb "vitess.io/vitess/go/vt/proto/vschema" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" @@ -110,10 +111,6 @@ func buildDDLPlans(ctx context.Context, sql string, ddlStatement sqlparser.DDLSt switch ddl := ddlStatement.(type) { case *sqlparser.AlterTable, *sqlparser.CreateTable, *sqlparser.TruncateTable: - err = checkFKError(vschema, ddlStatement) - if err != nil { - return nil, nil, err - } // For ALTER TABLE and TRUNCATE TABLE, the table must already exist // // For CREATE TABLE, the table may (in the case of --declarative) @@ -121,6 +118,10 @@ func buildDDLPlans(ctx context.Context, sql string, ddlStatement sqlparser.DDLSt // // We should find the target of the query from this tables location. destination, keyspace, err = findTableDestinationAndKeyspace(vschema, ddlStatement) + if err != nil { + return nil, nil, err + } + err = checkFKError(vschema, ddlStatement, keyspace) case *sqlparser.CreateView: destination, keyspace, err = buildCreateView(ctx, vschema, ddl, reservedVars, enableOnlineDDL, enableDirectDDL) case *sqlparser.AlterView: @@ -161,8 +162,12 @@ func buildDDLPlans(ctx context.Context, sql string, ddlStatement sqlparser.DDLSt }, nil } -func checkFKError(vschema plancontext.VSchema, ddlStatement sqlparser.DDLStatement) error { - if fkStrategyMap[vschema.ForeignKeyMode()] == fkDisallow { +func checkFKError(vschema plancontext.VSchema, ddlStatement sqlparser.DDLStatement, keyspace *vindexes.Keyspace) error { + fkMode, err := vschema.ForeignKeyMode(keyspace.Name) + if err != nil { + return err + } + if fkMode == vschemapb.Keyspace_FK_DISALLOW { fk := &fkContraint{} _ = sqlparser.Walk(fk.FkWalk, ddlStatement) if fk.found { diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index c0fab8a820c..057f2f5713d 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -426,6 +426,7 @@ func transformInsertPlan(ctx *plancontext.PlanningContext, op *operators.Route, return } } + return } diff --git a/go/vt/vtgate/planbuilder/operators/ast2op.go b/go/vt/vtgate/planbuilder/operators/ast2op.go index 59223f0e631..d81670a4921 100644 --- a/go/vt/vtgate/planbuilder/operators/ast2op.go +++ b/go/vt/vtgate/planbuilder/operators/ast2op.go @@ -20,6 +20,7 @@ import ( "fmt" "strconv" + vschemapb "vitess.io/vitess/go/vt/proto/vschema" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" @@ -260,6 +261,18 @@ func createOperatorFromInsert(ctx *plancontext.PlanningContext, ins *sqlparser.I Routing: routing, } + // Find the foreign key mode and store the ParentFKs that we need to verify. + ksMode, err := ctx.VSchema.ForeignKeyMode(vindexTable.Keyspace.Name) + if err != nil { + return nil, err + } + if ksMode == vschemapb.Keyspace_FK_MANAGED { + parentFKs := vindexTable.CrossShardParentFKs() + if len(parentFKs) > 0 { + return nil, vterrors.VT12002() + } + } + // Table column list is nil then add all the columns // If the column list is empty then add only the auto-inc column and // this happens on calling modifyForAutoinc diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 755a5fbf916..c8cdfbf7a0a 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -100,6 +100,21 @@ func TestPlan(t *testing.T) { testFile(t, "misc_cases.json", testOutputTempDir, vschemaWrapper, false) } +// TestForeignKeyPlanning tests the planning of foreign keys in a managed mode by Vitess. +func TestForeignKeyPlanning(t *testing.T) { + vschemaWrapper := &vschemaWrapper{ + v: loadSchema(t, "vschemas/schema.json", true), + // Set the keyspace with foreign keys enabled as the default. + keyspace: &vindexes.Keyspace{ + Name: "user_fk_allow", + Sharded: true, + }, + } + testOutputTempDir := makeTestOutput(t) + + testFile(t, "foreignkey_cases.json", testOutputTempDir, vschemaWrapper, false) +} + func TestSystemTables57(t *testing.T) { // first we move everything to use 5.7 logic servenv.SetMySQLServerVersionForTest("5.7") @@ -418,9 +433,31 @@ func loadSchema(t testing.TB, filename string, setCollation bool) *vindexes.VSch } } } + if vschema.Keyspaces["user_fk_allow"] != nil { + // FK from multicol_tbl2 referencing multicol_tbl1 that is shard scoped. + err = vschema.AddForeignKey("user_fk_allow", "multicol_tbl2", createFkDefinition([]string{"colb", "cola", "x", "colc", "y"}, "multicol_tbl1", []string{"colb", "cola", "y", "colc", "x"})) + require.NoError(t, err) + // FK from tbl2 referencing tbl1 that is shard scoped. + err = vschema.AddForeignKey("user_fk_allow", "tbl2", createFkDefinition([]string{"col2"}, "tbl1", []string{"col1"})) + require.NoError(t, err) + // FK from tbl3 referencing tbl1 that is not shard scoped. + err = vschema.AddForeignKey("user_fk_allow", "tbl3", createFkDefinition([]string{"coly"}, "tbl1", []string{"col1"})) + require.NoError(t, err) + } return vschema } +// createFkDefinition is a helper function to create a Foreign key definition struct from the columns used in it provided as list of strings. +func createFkDefinition(childCols []string, parentTableName string, parentCols []string) *sqlparser.ForeignKeyDefinition { + return &sqlparser.ForeignKeyDefinition{ + Source: sqlparser.MakeColumns(childCols...), + ReferenceDefinition: &sqlparser.ReferenceDefinition{ + ReferencedTable: sqlparser.NewTableName(parentTableName), + ReferencedColumns: sqlparser.MakeColumns(parentCols...), + }, + } +} + var _ plancontext.VSchema = (*vschemaWrapper)(nil) type vschemaWrapper struct { @@ -507,8 +544,12 @@ func (vw *vschemaWrapper) ConnCollation() collations.ID { func (vw *vschemaWrapper) PlannerWarning(_ string) { } -func (vw *vschemaWrapper) ForeignKeyMode() string { - return "allow" +func (vw *vschemaWrapper) ForeignKeyMode(keyspace string) (vschemapb.Keyspace_ForeignKeyMode, error) { + defaultFkMode := vschemapb.Keyspace_FK_UNMANAGED + if vw.v.Keyspaces[keyspace] != nil && vw.v.Keyspaces[keyspace].ForeignKeyMode != vschemapb.Keyspace_FK_DEFAULT { + return vw.v.Keyspaces[keyspace].ForeignKeyMode, nil + } + return defaultFkMode, nil } func (vw *vschemaWrapper) AllKeyspace() ([]*vindexes.Keyspace, error) { diff --git a/go/vt/vtgate/planbuilder/plancontext/vschema.go b/go/vt/vtgate/planbuilder/plancontext/vschema.go index f288a9bd95c..fc5ee6d9207 100644 --- a/go/vt/vtgate/planbuilder/plancontext/vschema.go +++ b/go/vt/vtgate/planbuilder/plancontext/vschema.go @@ -55,7 +55,7 @@ type VSchema interface { PlannerWarning(message string) // ForeignKeyMode returns the foreign_key flag value - ForeignKeyMode() string + ForeignKeyMode(keyspace string) (vschemapb.Keyspace_ForeignKeyMode, error) // GetVSchema returns the latest cached vindexes.VSchema GetVSchema() *vindexes.VSchema diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json new file mode 100644 index 00000000000..3d244086667 --- /dev/null +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -0,0 +1,57 @@ +[ + { + "comment": "Insertion in a table with cross-shard foreign keys disallowed", + "query": "insert into tbl3 (col3, coly) values (1, 3)", + "plan": "VT12002: unsupported: cross-shard foreign keys" + }, + { + "comment": "Insertion in a table with shard-scoped foreign keys is allowed", + "query": "insert into tbl2 (col2, coly) values (1, 3)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into tbl2 (col2, coly) values (1, 3)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Sharded", + "Keyspace": { + "Name": "user_fk_allow", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into tbl2(col2, coly) values (:_col2_0, 3)", + "TableName": "tbl2", + "VindexValues": { + "hash_vin": "INT64(1)" + } + }, + "TablesUsed": [ + "user_fk_allow.tbl2" + ] + } + }, + { + "comment": "Insertion in a table with shard-scoped multiple column foreign key is allowed", + "query": "insert into multicol_tbl2 (cola, colb, colc) values (1, 2, 3)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into multicol_tbl2 (cola, colb, colc) values (1, 2, 3)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Sharded", + "Keyspace": { + "Name": "user_fk_allow", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into multicol_tbl2(cola, colb, colc) values (:_cola_0, :_colb_0, :_colc_0)", + "TableName": "multicol_tbl2", + "VindexValues": { + "multicolIdx": "INT64(1), INT64(2), INT64(3)" + } + }, + "TablesUsed": [ + "user_fk_allow.multicol_tbl2" + ] + } + } +] \ No newline at end of file diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json index 9f9ea97200f..1d23f669795 100644 --- a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json +++ b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json @@ -602,6 +602,69 @@ ] } } + }, + "user_fk_allow": { + "sharded": true, + "foreignKeyMode": "FK_MANAGED", + "vindexes": { + "hash_vin": { + "type": "hash_test", + "owner": "user" + }, + "multicolIdx": { + "type": "multiCol_test" + } + }, + "tables": { + "multicol_tbl1": { + "column_vindexes": [ + { + "columns": [ + "cola", + "colb", + "colc" + ], + "name": "multicolIdx" + } + ] + }, + "multicol_tbl2": { + "column_vindexes": [ + { + "columns": [ + "cola", + "colb", + "colc" + ], + "name": "multicolIdx" + } + ] + }, + "tbl1": { + "column_vindexes": [ + { + "column": "col1", + "name": "hash_vin" + } + ] + }, + "tbl2": { + "column_vindexes": [ + { + "column": "col2", + "name": "hash_vin" + } + ] + }, + "tbl3": { + "column_vindexes": [ + { + "column": "col3", + "name": "hash_vin" + } + ] + } + } } } } diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index 3d4778d430d..f36de3d87ae 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -1011,8 +1011,15 @@ func (vc *vcursorImpl) PlannerWarning(message string) { } // ForeignKeyMode implements the VCursor interface -func (vc *vcursorImpl) ForeignKeyMode() string { - return strings.ToLower(foreignKeyMode) +func (vc *vcursorImpl) ForeignKeyMode(keyspace string) (vschemapb.Keyspace_ForeignKeyMode, error) { + if strings.ToLower(foreignKeyMode) == "disallow" { + return vschemapb.Keyspace_FK_DISALLOW, nil + } + ks := vc.vschema.Keyspaces[keyspace] + if ks == nil { + return 0, vterrors.VT14004(keyspace) + } + return ks.ForeignKeyMode, nil } // ParseDestinationTarget parses destination target string and sets default keyspace if possible. diff --git a/go/vt/vtgate/vindexes/foreign_keys.go b/go/vt/vtgate/vindexes/foreign_keys.go new file mode 100644 index 00000000000..dca74d163d7 --- /dev/null +++ b/go/vt/vtgate/vindexes/foreign_keys.go @@ -0,0 +1,166 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "encoding/json" + "fmt" + + "vitess.io/vitess/go/vt/sqlparser" +) + +// ParentFKInfo contains the parent foreign key info for the table. +type ParentFKInfo struct { + Table *Table + ParentColumns sqlparser.Columns + ChildColumns sqlparser.Columns +} + +// MarshalJSON returns a JSON representation of ParentFKInfo. +func (fk *ParentFKInfo) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Name string `json:"parent_table"` + ParentColumns sqlparser.Columns `json:"parent_columns"` + ChildColumns sqlparser.Columns `json:"child_columns"` + }{ + Name: fk.Table.Name.String(), + ChildColumns: fk.ChildColumns, + ParentColumns: fk.ParentColumns, + }) +} + +// NewParentFkInfo creates a new ParentFKInfo. +func NewParentFkInfo(parentTbl *Table, fkDef *sqlparser.ForeignKeyDefinition) ParentFKInfo { + return ParentFKInfo{ + Table: parentTbl, + ChildColumns: fkDef.Source, + ParentColumns: fkDef.ReferenceDefinition.ReferencedColumns, + } +} + +// ChildFKInfo contains the child foreign key info for the table. +type ChildFKInfo struct { + Table *Table + ChildColumns sqlparser.Columns + ParentColumns sqlparser.Columns + Match sqlparser.MatchAction + OnDelete sqlparser.ReferenceAction + OnUpdate sqlparser.ReferenceAction +} + +// MarshalJSON returns a JSON representation of ChildFKInfo. +func (fk *ChildFKInfo) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Name string `json:"child_table"` + ChildColumns sqlparser.Columns `json:"child_columns"` + ParentColumns sqlparser.Columns `json:"parent_columns"` + }{ + Name: fk.Table.Name.String(), + ChildColumns: fk.ChildColumns, + ParentColumns: fk.ParentColumns, + }) +} + +// NewChildFkInfo creates a new ChildFKInfo. +func NewChildFkInfo(childTbl *Table, fkDef *sqlparser.ForeignKeyDefinition) ChildFKInfo { + return ChildFKInfo{ + Table: childTbl, + ChildColumns: fkDef.Source, + ParentColumns: fkDef.ReferenceDefinition.ReferencedColumns, + Match: fkDef.ReferenceDefinition.Match, + OnDelete: fkDef.ReferenceDefinition.OnDelete, + OnUpdate: fkDef.ReferenceDefinition.OnUpdate, + } +} + +// AddForeignKey is for testing only. +func (vschema *VSchema) AddForeignKey(ksname, childTableName string, fkConstraint *sqlparser.ForeignKeyDefinition) error { + ks, ok := vschema.Keyspaces[ksname] + if !ok { + return fmt.Errorf("keyspace %s not found in vschema", ksname) + } + cTbl, ok := ks.Tables[childTableName] + if !ok { + return fmt.Errorf("child table %s not found in keyspace %s", childTableName, ksname) + } + parentTableName := fkConstraint.ReferenceDefinition.ReferencedTable.Name.String() + pTbl, ok := ks.Tables[parentTableName] + if !ok { + return fmt.Errorf("parent table %s not found in keyspace %s", parentTableName, ksname) + } + pTbl.ChildForeignKeys = append(pTbl.ChildForeignKeys, NewChildFkInfo(cTbl, fkConstraint)) + cTbl.ParentForeignKeys = append(cTbl.ParentForeignKeys, NewParentFkInfo(pTbl, fkConstraint)) + return nil +} + +// CrossShardParentFKs returns all the parent fk constraints on this table that are not shard scoped. +func (t *Table) CrossShardParentFKs() (crossShardFKs []ParentFKInfo) { + if len(t.ParentForeignKeys) == 0 { + return + } + for _, fk := range t.ParentForeignKeys { + // If the keyspaces are different, then the fk definition + // is going to go across shards. + if fk.Table.Keyspace.Name != t.Keyspace.Name { + crossShardFKs = append(crossShardFKs, fk) + continue + } + // If the keyspaces match and they are unsharded, then the fk defintion + // is shard-scoped. + if !t.Keyspace.Sharded { + continue + } + + // If the primary vindexes don't match between the parent and child table, + // we cannot infer that the fk constraint in shard scoped. + primaryVindex := t.ColumnVindexes[0] + if fk.Table.ColumnVindexes[0].Vindex != primaryVindex.Vindex { + crossShardFKs = append(crossShardFKs, fk) + continue + } + + childFkContatined, childFkIndexes := fk.ChildColumns.Indexes(primaryVindex.Columns) + if !childFkContatined { + // PrimaryVindex is not part of the foreign key constraint on the children side. + // So it is a cross-shard foreign key. + crossShardFKs = append(crossShardFKs, fk) + continue + } + + // We need to run the same check for the parent columns. + parentFkContatined, parentFkIndexes := fk.ParentColumns.Indexes(fk.Table.ColumnVindexes[0].Columns) + if !parentFkContatined { + crossShardFKs = append(crossShardFKs, fk) + continue + } + + // Both the child and parent table contain the foreign key and that the vindexes are the same, + // now we need to make sure, that the indexes of both match. + // For example, consider the following tables, + // t1 (primary vindex (x,y)) + // t2 (primary vindex (a,b)) + // If we have a foreign key constraint from t1(x,y) to t2(b,a), then they are not shard scoped. + // Let's say in t1, (1,3) will be in -80 and (3,1) will be in 80-, then in t2 (1,3) will end up in 80-. + for i := range parentFkIndexes { + if parentFkIndexes[i] != childFkIndexes[i] { + crossShardFKs = append(crossShardFKs, fk) + break + } + } + } + return +} diff --git a/go/vt/vtgate/vindexes/foreign_keys_test.go b/go/vt/vtgate/vindexes/foreign_keys_test.go new file mode 100644 index 00000000000..b56bdf2f062 --- /dev/null +++ b/go/vt/vtgate/vindexes/foreign_keys_test.go @@ -0,0 +1,156 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/vt/sqlparser" +) + +var ( + uks = &Keyspace{Name: "uks"} + uks2 = &Keyspace{Name: "uks2"} + sks = &Keyspace{Name: "sks", Sharded: true} +) + +// TestTable_CrossShardParentFKs tests the functionality of the method CrossShardParentFKs. +func TestTable_CrossShardParentFKs(t *testing.T) { + col1Vindex := &ColumnVindex{ + Name: "v1", + Vindex: binVindex, + Columns: sqlparser.MakeColumns("col1"), + } + col4DiffVindex := &ColumnVindex{ + Name: "v2", + Vindex: binOnlyVindex, + Columns: sqlparser.MakeColumns("col4"), + } + col123Vindex := &ColumnVindex{ + Name: "v2", + Vindex: binVindex, + Columns: sqlparser.MakeColumns("col1", "col2", "col3"), + } + col456Vindex := &ColumnVindex{ + Name: "v2", + Vindex: binVindex, + Columns: sqlparser.MakeColumns("col4", "col5", "col6"), + } + + unshardedTbl := &Table{ + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: uks2, + } + shardedSingleColTblWithDiffVindex := &Table{ + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: sks, + ColumnVindexes: []*ColumnVindex{col4DiffVindex}, + } + shardedMultiColTbl := &Table{ + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: sks, + ColumnVindexes: []*ColumnVindex{col456Vindex}, + } + + tests := []struct { + name string + table *Table + wantCrossShardFKTables []string + }{{ + name: "No Parent FKs", + table: &Table{ + ColumnVindexes: []*ColumnVindex{col1Vindex}, + Keyspace: sks, + }, + wantCrossShardFKTables: []string{}, + }, { + name: "Unsharded keyspace", + table: &Table{ + ColumnVindexes: []*ColumnVindex{col1Vindex}, + Keyspace: uks2, + ParentForeignKeys: []ParentFKInfo{pkInfo(unshardedTbl, []string{"col4"}, []string{"col1"})}, + }, + wantCrossShardFKTables: []string{}, + }, { + name: "Keyspaces don't match", // parent table is on uks2 + table: &Table{ + Keyspace: uks, + ParentForeignKeys: []ParentFKInfo{pkInfo(unshardedTbl, []string{"col4"}, []string{"col1"})}, + }, + wantCrossShardFKTables: []string{"t1"}, + }, { + name: "Column Vindexes don't match", // primary vindexes on different vindex type + table: &Table{ + Keyspace: sks, + ColumnVindexes: []*ColumnVindex{col1Vindex}, + ParentForeignKeys: []ParentFKInfo{pkInfo(shardedSingleColTblWithDiffVindex, []string{"col4"}, []string{"col1"})}, + }, + wantCrossShardFKTables: []string{"t1"}, + }, { + name: "child table foreign key does not contain primary vindex columns", + table: &Table{ + Keyspace: sks, + ColumnVindexes: []*ColumnVindex{col123Vindex}, + ParentForeignKeys: []ParentFKInfo{pkInfo(shardedMultiColTbl, []string{"col4", "col5", "col6"}, []string{"col3", "col9", "col1"})}, + }, + wantCrossShardFKTables: []string{"t1"}, + }, { + name: "Parent FK doesn't contain primary vindex", + table: &Table{ + Keyspace: sks, + ColumnVindexes: []*ColumnVindex{col123Vindex}, + ParentForeignKeys: []ParentFKInfo{pkInfo(shardedMultiColTbl, []string{"col4", "col9", "col6"}, []string{"col1", "col2", "col3"})}, + }, + wantCrossShardFKTables: []string{"t1"}, + }, { + name: "Indexes of the two FKs with column vindexes don't line up", + table: &Table{ + Keyspace: sks, + ColumnVindexes: []*ColumnVindex{col123Vindex}, + ParentForeignKeys: []ParentFKInfo{pkInfo(shardedMultiColTbl, []string{"col4", "col9", "col5", "col6"}, []string{"col1", "col2", "col3", "col9"})}, + }, + wantCrossShardFKTables: []string{"t1"}, + }, { + name: "Shard scoped foreign key constraint", + table: &Table{ + Keyspace: sks, + ColumnVindexes: []*ColumnVindex{col123Vindex}, + ParentForeignKeys: []ParentFKInfo{pkInfo(shardedMultiColTbl, []string{"col4", "col9", "col5", "col6", "colc"}, []string{"col1", "cola", "col2", "col3", "colb"})}, + }, + wantCrossShardFKTables: []string{}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + crossShardFks := tt.table.CrossShardParentFKs() + var crossShardFkTables []string + for _, fk := range crossShardFks { + crossShardFkTables = append(crossShardFkTables, fk.Table.Name.String()) + } + require.ElementsMatch(t, tt.wantCrossShardFKTables, crossShardFkTables) + }) + } +} + +func pkInfo(parentTable *Table, pCols []string, cCols []string) ParentFKInfo { + return ParentFKInfo{ + Table: parentTable, + ParentColumns: sqlparser.MakeColumns(pCols...), + ChildColumns: sqlparser.MakeColumns(cCols...), + } +} diff --git a/go/vt/vtgate/vindexes/vschema.go b/go/vt/vtgate/vindexes/vschema.go index a7e1b2fe2ad..6edd97aeeb5 100644 --- a/go/vt/vtgate/vindexes/vschema.go +++ b/go/vt/vtgate/vindexes/vschema.go @@ -139,70 +139,6 @@ type ColumnVindex struct { backfill bool } -// ParentFKInfo contains the parent foreign key info for the table. -type ParentFKInfo struct { - Table *Table - ParentColumns sqlparser.Columns - ChildColumns sqlparser.Columns -} - -// MarshalJSON returns a JSON representation of ParentFKInfo. -func (fk *ParentFKInfo) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - Name string `json:"parent_table"` - ParentColumns sqlparser.Columns `json:"parent_columns"` - ChildColumns sqlparser.Columns `json:"child_columns"` - }{ - Name: fk.Table.Name.String(), - ChildColumns: fk.ChildColumns, - ParentColumns: fk.ParentColumns, - }) -} - -// NewParentFkInfo creates a new ParentFKInfo. -func NewParentFkInfo(parentTbl *Table, fkDef *sqlparser.ForeignKeyDefinition) ParentFKInfo { - return ParentFKInfo{ - Table: parentTbl, - ChildColumns: fkDef.Source, - ParentColumns: fkDef.ReferenceDefinition.ReferencedColumns, - } -} - -// ChildFKInfo contains the child foreign key info for the table. -type ChildFKInfo struct { - Table *Table - ChildColumns sqlparser.Columns - ParentColumns sqlparser.Columns - Match sqlparser.MatchAction - OnDelete sqlparser.ReferenceAction - OnUpdate sqlparser.ReferenceAction -} - -// MarshalJSON returns a JSON representation of ChildFKInfo. -func (fk *ChildFKInfo) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - Name string `json:"child_table"` - ChildColumns sqlparser.Columns `json:"child_columns"` - ParentColumns sqlparser.Columns `json:"parent_columns"` - }{ - Name: fk.Table.Name.String(), - ChildColumns: fk.ChildColumns, - ParentColumns: fk.ParentColumns, - }) -} - -// NewChildFkInfo creates a new ChildFKInfo. -func NewChildFkInfo(childTbl *Table, fkDef *sqlparser.ForeignKeyDefinition) ChildFKInfo { - return ChildFKInfo{ - Table: childTbl, - ChildColumns: fkDef.Source, - ParentColumns: fkDef.ReferenceDefinition.ReferencedColumns, - Match: fkDef.ReferenceDefinition.Match, - OnDelete: fkDef.ReferenceDefinition.OnDelete, - OnUpdate: fkDef.ReferenceDefinition.OnUpdate, - } -} - // TableInfo contains column and foreign key info for a table. type TableInfo struct { Columns []Column @@ -402,26 +338,6 @@ func replaceDefaultForeignKeyMode(fkMode vschemapb.Keyspace_ForeignKeyMode) vsch return fkMode } -// addForeignKey is for testing only. -func (vschema *VSchema) addForeignKey(ksname, childTableName string, fkConstraint *sqlparser.ForeignKeyDefinition) error { - ks, ok := vschema.Keyspaces[ksname] - if !ok { - return fmt.Errorf("keyspace %s not found in vschema", ksname) - } - cTbl, ok := ks.Tables[childTableName] - if !ok { - return fmt.Errorf("child table %s not found in keyspace %s", childTableName, ksname) - } - parentTableName := fkConstraint.ReferenceDefinition.ReferencedTable.Name.String() - pTbl, ok := ks.Tables[parentTableName] - if !ok { - return fmt.Errorf("parent table %s not found in keyspace %s", parentTableName, ksname) - } - pTbl.ChildForeignKeys = append(pTbl.ChildForeignKeys, NewChildFkInfo(cTbl, fkConstraint)) - cTbl.ParentForeignKeys = append(cTbl.ParentForeignKeys, NewParentFkInfo(pTbl, fkConstraint)) - return nil -} - func (vschema *VSchema) AddView(ksname string, viewName, query string) error { ks, ok := vschema.Keyspaces[ksname] if !ok { diff --git a/go/vt/vtgate/vindexes/vschema_test.go b/go/vt/vtgate/vindexes/vschema_test.go index 1e21235dc0e..58874967d49 100644 --- a/go/vt/vtgate/vindexes/vschema_test.go +++ b/go/vt/vtgate/vindexes/vschema_test.go @@ -411,7 +411,7 @@ func TestVSchemaForeignKeys(t *testing.T) { require.NoError(t, vschema.Keyspaces["main"].Error) // add fk containst a keyspace. - vschema.addForeignKey("main", "t1", &sqlparser.ForeignKeyDefinition{ + vschema.AddForeignKey("main", "t1", &sqlparser.ForeignKeyDefinition{ Source: sqlparser.Columns{sqlparser.NewIdentifierCI("c2")}, ReferenceDefinition: &sqlparser.ReferenceDefinition{ ReferencedTable: sqlparser.NewTableName("t1"), diff --git a/test/config.json b/test/config.json index c0f0b0825f8..b46152462c4 100644 --- a/test/config.json +++ b/test/config.json @@ -842,6 +842,15 @@ "RetryMax": 1, "Tags": [] }, + "vtgate_foreignkey": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/foreignkey"], + "Command": [], + "Manual": false, + "Shard": "vtgate_gen4", + "RetryMax": 2, + "Tags": [] + }, "vtgate_gen4": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/gen4"],