diff --git a/go/vt/vtgate/engine/fk_verify.go b/go/vt/vtgate/engine/fk_verify.go index 408317bb37c..0ae796b2464 100644 --- a/go/vt/vtgate/engine/fk_verify.go +++ b/go/vt/vtgate/engine/fk_verify.go @@ -152,6 +152,7 @@ func (f *FkVerify) Inputs() ([]Primitive, []map[string]any) { inputName: fmt.Sprintf("VerifyParent-%d", idx+1), "BvName": parent.BvName, "Cols": parent.Cols, + "Values": parent.Values, }) inputs = append(inputs, parent.Exec) } diff --git a/go/vt/vtgate/planbuilder/delete.go b/go/vt/vtgate/planbuilder/delete.go index 034d1fa020f..2c12f92b72a 100644 --- a/go/vt/vtgate/planbuilder/delete.go +++ b/go/vt/vtgate/planbuilder/delete.go @@ -57,7 +57,7 @@ func gen4DeleteStmtPlanner( } if ks, tables := ctx.SemTable.SingleUnshardedKeyspace(); ks != nil { - if fkManagementNotRequired(vschema, tables) { + if fkManagementNotRequiredForDelete(vschema, tables) { plan := deleteUnshardedShortcut(deleteStmt, ks, tables) plan = pushCommentDirectivesOnPlan(plan, deleteStmt) return newPlanResult(plan.Primitive(), operators.QualifiedTables(ks, tables)...), nil @@ -94,7 +94,7 @@ func gen4DeleteStmtPlanner( return newPlanResult(plan.Primitive(), operators.TablesUsed(op)...), nil } -func fkManagementNotRequired(vschema plancontext.VSchema, vTables []*vindexes.Table) bool { +func fkManagementNotRequiredForDelete(vschema plancontext.VSchema, vTables []*vindexes.Table) bool { // Find the foreign key mode and check for any managed child foreign keys. for _, vTable := range vTables { ksMode, err := vschema.ForeignKeyMode(vTable.Keyspace.Name) diff --git a/go/vt/vtgate/planbuilder/fk_verify.go b/go/vt/vtgate/planbuilder/fk_verify.go new file mode 100644 index 00000000000..c3b1545f32f --- /dev/null +++ b/go/vt/vtgate/planbuilder/fk_verify.go @@ -0,0 +1,78 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package planbuilder + +import ( + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/engine" + "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" + "vitess.io/vitess/go/vt/vtgate/semantics" +) + +var _ logicalPlan = (*fkVerify)(nil) + +// fkVerify is the logicalPlan for engine.FkVerify. +type fkVerify struct { + input logicalPlan + verify []*engine.FkParent +} + +// newFkVerify builds a new fkVerify. +func newFkVerify(input logicalPlan, verify []*engine.FkParent) *fkVerify { + return &fkVerify{ + input: input, + verify: verify, + } +} + +// Primitive implements the logicalPlan interface +func (fkc *fkVerify) Primitive() engine.Primitive { + return &engine.FkVerify{ + Exec: fkc.input.Primitive(), + Verify: fkc.verify, + } +} + +// Wireup implements the logicalPlan interface +func (fkc *fkVerify) Wireup(ctx *plancontext.PlanningContext) error { + return fkc.input.Wireup(ctx) +} + +// Rewrite implements the logicalPlan interface +func (fkc *fkVerify) Rewrite(inputs ...logicalPlan) error { + if len(inputs) != 1 { + return vterrors.VT13001("fkVerify: wrong number of inputs") + } + fkc.input = inputs[0] + return nil +} + +// ContainsTables implements the logicalPlan interface +func (fkc *fkVerify) ContainsTables() semantics.TableSet { + return fkc.input.ContainsTables() +} + +// Inputs implements the logicalPlan interface +func (fkc *fkVerify) Inputs() []logicalPlan { + return []logicalPlan{fkc.input} +} + +// OutputColumns implements the logicalPlan interface +func (fkc *fkVerify) OutputColumns() []sqlparser.SelectExpr { + return nil +} diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 283ee147ba4..d9e5cc3d2b9 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -67,17 +67,52 @@ func transformToLogicalPlan(ctx *plancontext.PlanningContext, op ops.Operator) ( return transformDistinct(ctx, op) case *operators.FkCascade: return transformFkCascade(ctx, op) + case *operators.FkVerify: + return transformFkVerify(ctx, op) } return nil, vterrors.VT13001(fmt.Sprintf("unknown type encountered: %T (transformToLogicalPlan)", op)) } +func transformFkVerify(ctx *plancontext.PlanningContext, fkv *operators.FkVerify) (logicalPlan, error) { + inputLP, err := transformToLogicalPlan(ctx, fkv.Input) + if err != nil { + return nil, err + } + + // Once we have the input logical plan, we can create the primitives for the verification operators. + // For all of these, we don't need the semTable anymore. We set it to nil, to avoid using an incorrect one. + ctx.SemTable = nil + + // Go over the children and convert them to Primitives too. + var parents []*engine.FkParent + for _, verify := range fkv.Verify { + verifyLP, err := transformToLogicalPlan(ctx, verify.Op) + if err != nil { + return nil, err + } + err = verifyLP.Wireup(ctx) + if err != nil { + return nil, err + } + verifyEngine := verifyLP.Primitive() + parents = append(parents, &engine.FkParent{ + BvName: verify.BvName, + Values: verify.Values, + Cols: verify.Cols, + Exec: verifyEngine, + }) + } + + return newFkVerify(inputLP, parents), nil +} + // transformFkCascade transforms a FkCascade operator into a logical plan. func transformFkCascade(ctx *plancontext.PlanningContext, fkc *operators.FkCascade) (logicalPlan, error) { // We convert the parent operator to a logical plan. parentLP, err := transformToLogicalPlan(ctx, fkc.Parent) if err != nil { - return nil, nil + return nil, err } // Once we have the parent logical plan, we can create the selection logical plan and the primitives for the children operators. diff --git a/go/vt/vtgate/planbuilder/operators/ast2op.go b/go/vt/vtgate/planbuilder/operators/ast2op.go index 8e1e76c65c6..39601bbd806 100644 --- a/go/vt/vtgate/planbuilder/operators/ast2op.go +++ b/go/vt/vtgate/planbuilder/operators/ast2op.go @@ -20,6 +20,7 @@ import ( "fmt" "strconv" + "vitess.io/vitess/go/mysql/collations" vschemapb "vitess.io/vitess/go/vt/proto/vschema" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -132,6 +133,130 @@ func createOperatorFromInsert(ctx *plancontext.PlanningContext, ins *sqlparser.I return nil, err } + insOp, err := createInsertOperator(ctx, ins, routing, vindexTable) + if err != nil { + return nil, err + } + + // Find the foreign key mode and store the ParentFKs that we need to verify. + ksMode, err := ctx.VSchema.ForeignKeyMode(vindexTable.Keyspace.Name) + if err != nil { + return nil, err + } + if ksMode != vschemapb.Keyspace_FK_MANAGED { + return insOp, nil + } + parentFKs := vindexTable.ParentFKsNeedsHandling() + if len(parentFKs) == 0 { + return insOp, nil + } + // TODO(@GuptaManan100, @harshit-gangal): Reject some queries that are unsupported + // return nil, vterrors.VT12002() + + insValues, isValues := ins.Rows.(sqlparser.Values) + if !isValues { + // TODO(@GuptaManan100, @harshit-gangal) - Fix error message + return nil, vterrors.VT12001() + } + + // For each fk we create a FkParent + var verify []*FkParent + for _, fk := range parentFKs { + // Verification query looks like - + // SELECT distinct from parent where () in ::fkv_vals + bvName := ctx.ReservedVars.ReserveVariable("fkv_vals") + + var valTuple sqlparser.ValTuple + var selExprs sqlparser.SelectExprs + for _, column := range fk.ParentColumns { + valTuple = append(valTuple, sqlparser.NewColName(column.String())) + selExprs = append(selExprs, aeWrap(sqlparser.NewColName(column.String()))) + } + + selStmt := &sqlparser.Select{ + Distinct: true, + SelectExprs: selExprs, + From: []sqlparser.TableExpr{ + sqlparser.NewAliasedTableExpr(fk.Table.GetTableName(), ""), + }, + Where: &sqlparser.Where{ + Type: sqlparser.WhereClause, + Expr: &sqlparser.ComparisonExpr{ + Operator: sqlparser.InOp, + Left: valTuple, + Right: sqlparser.NewListArg(bvName), + }, + }, + } + selOp, err := createOpFromStmt(ctx, selStmt) + if err != nil { + return nil, err + } + + var checkCols []engine.CheckCol + for idx, column := range fk.ChildColumns { + checkCol := engine.CheckCol{ + Col: idx, + } + found := false + for _, col := range vindexTable.Columns { + if column.Equal(col.Name) { + found = true + checkCol.Type = col.Type + if col.CollationName != "" { + // TODO(@GuptaManan100, @harshit-gangal) - Convert Collation from string to ID. + } else { + checkCol.Collation = collations.CollationBinaryID + } + break + } + } + if !found { + // TODO(@GuptaManan100, @harshit-gangal) - Fix error message + return nil, vterrors.VT12002() + } + checkCols = append(checkCols, checkCol) + } + + var colIdxs []int + for _, column := range fk.ChildColumns { + found := false + for idx, col := range ins.Columns { + if column.Equal(col) { + found = true + colIdxs = append(colIdxs, idx) + break + } + } + if !found { + // TODO(@GuptaManan100, @harshit-gangal) - Fix error message + return nil, vterrors.VT12002() + } + } + + var vals []sqlparser.Exprs + for _, tuple := range insValues { + var exprs sqlparser.Exprs + for _, idx := range colIdxs { + exprs = append(exprs, tuple[idx]) + } + vals = append(vals, exprs) + } + verify = append(verify, &FkParent{ + Values: vals, + Cols: checkCols, + BvName: bvName, + Op: selOp, + }) + } + + return &FkVerify{ + Input: insOp, + Verify: verify, + }, nil +} + +func createInsertOperator(ctx *plancontext.PlanningContext, ins *sqlparser.Insert, routing Routing, vindexTable *vindexes.Table) (ops.Operator, error) { if _, target := routing.(*TargetedRouting); target { return nil, vterrors.VT12001("INSERT with a target destination") } @@ -145,18 +270,6 @@ func createOperatorFromInsert(ctx *plancontext.PlanningContext, ins *sqlparser.I Routing: routing, } - // Find the foreign key mode and store the ParentFKs that we need to verify. - ksMode, err := ctx.VSchema.ForeignKeyMode(vindexTable.Keyspace.Name) - if err != nil { - return nil, err - } - if ksMode == vschemapb.Keyspace_FK_MANAGED { - parentFKs := vindexTable.ParentFKsNeedsHandling() - if len(parentFKs) > 0 { - return nil, vterrors.VT12002() - } - } - // Table column list is nil then add all the columns // If the column list is empty then add only the auto-inc column and // this happens on calling modifyForAutoinc diff --git a/go/vt/vtgate/planbuilder/operators/fk_verify.go b/go/vt/vtgate/planbuilder/operators/fk_verify.go new file mode 100644 index 00000000000..0d7d87d67e7 --- /dev/null +++ b/go/vt/vtgate/planbuilder/operators/fk_verify.go @@ -0,0 +1,100 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operators + +import ( + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vtgate/engine" + "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" +) + +type FkParent struct { + Values []sqlparser.Exprs + Cols []engine.CheckCol + BvName string + + Op ops.Operator +} + +// FkVerify is used to represent a foreign key verification operation +// as an operator. This operator is created for DML queries that require +// verifications on the existence of the rows in the parent table (for example, INSERT and UPDATE). +type FkVerify struct { + Verify []*FkParent + Input ops.Operator + + noColumns + noPredicates +} + +var _ ops.Operator = (*FkVerify)(nil) + +// Inputs implements the Operator interface +func (fkv *FkVerify) Inputs() []ops.Operator { + var inputs []ops.Operator + inputs = append(inputs, fkv.Input) + for _, child := range fkv.Verify { + inputs = append(inputs, child.Op) + } + return inputs +} + +// SetInputs implements the Operator interface +func (fkv *FkVerify) SetInputs(operators []ops.Operator) { + if len(operators) < 1 { + panic("incorrect count of inputs for FkVerify") + } + fkv.Input = operators[0] + for idx, operator := range operators { + if idx < 1 { + continue + } + fkv.Verify[idx-1].Op = operator + } +} + +// Clone implements the Operator interface +func (fkv *FkVerify) Clone(inputs []ops.Operator) ops.Operator { + if len(inputs) < 1 { + panic("incorrect count of inputs for FkVerify") + } + newFkv := &FkVerify{ + Input: inputs[0], + } + for idx, operator := range inputs { + if idx < 1 { + continue + } + newFkv.Verify = append(newFkv.Verify, &FkParent{ + Values: fkv.Verify[idx-1].Values, + BvName: fkv.Verify[idx-1].BvName, + Cols: fkv.Verify[idx-1].Cols, + Op: operator, + }) + } + return newFkv +} + +// GetOrdering implements the Operator interface +func (fkv *FkVerify) GetOrdering() ([]ops.OrderBy, error) { + return nil, nil +} + +// ShortDescription implements the Operator interface +func (fkv *FkVerify) ShortDescription() string { + return "FkVerify" +} diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 7d54d01b565..4fdb14359a3 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -42,7 +42,6 @@ import ( "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vtgate/engine" - oprewriters "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/rewrite" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" "vitess.io/vitess/go/vt/vtgate/semantics" "vitess.io/vitess/go/vt/vtgate/vindexes" @@ -195,8 +194,8 @@ func TestViews(t *testing.T) { } func TestOne(t *testing.T) { - reset := oprewriters.EnableDebugPrinting() - defer reset() + //reset := oprewriters.EnableDebugPrinting() + //defer reset() lv := loadSchema(t, "vschemas/schema.json", true) setFks(t, lv) diff --git a/go/vt/vtgate/planbuilder/testdata/onecase.json b/go/vt/vtgate/planbuilder/testdata/onecase.json index da7543f706a..70dc33f5c3e 100644 --- a/go/vt/vtgate/planbuilder/testdata/onecase.json +++ b/go/vt/vtgate/planbuilder/testdata/onecase.json @@ -1,9 +1,86 @@ [ { "comment": "Add your test case here for debugging and run go test -run=One.", - "query": "", + "query": "insert into tbl3 (col3, coly) values (1,2),(3,4),(:v1,:v2)", "plan": { - + "QueryType": "INSERT", + "Original": "insert into tbl3 (col3, coly) values (1,2),(3,4),(:v1,:v2)", + "Instructions": { + "OperatorType": "FKVerify", + "Child": { + "OperatorType": "Insert", + "Variant": "Sharded", + "Keyspace": { + "Name": "sharded_fk_allow", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into tbl3(col3, coly) values (:_col3_0, 2), (:_col3_1, 4), (:_col3_2, :v2)", + "TableName": "tbl3", + "VindexValues": { + "hash_vin": "INT64(1), INT64(3), :v1" + } + }, + "Parent": [ + { + "OperatorType": "FkVerifyParent", + "BvName": "fkv_vals", + "Cols": [ + { + "Col": 0, + "WsCol": null, + "Type": 259, + "Collation": 63 + } + ], + "Values": [ + [ + { + "Type": 1, + "Val": "2" + } + ], + [ + { + "Type": 1, + "Val": "4" + } + ], + [ + { + "Name": "v2", + "Type": -1 + } + ] + ], + "Inputs": [ + { + "OperatorType": "Distinct", + "Collations": [ + "(0:1)" + ], + "ResultColumns": 1, + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "sharded_fk_allow", + "Sharded": true + }, + "FieldQuery": "select t1col1, weight_string(t1col1) from tbl3 where 1 != 1", + "Query": "select distinct t1col1, weight_string(t1col1) from tbl3 where (t1col1) in ::fkv_vals", + "Table": "tbl3" + } + ] + } + ] + } + ] + }, + "TablesUsed": [ + "sharded_fk_allow.tbl3" + ] } } ] \ No newline at end of file diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json index e7f38d88404..cdaf83004b1 100644 --- a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json +++ b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json @@ -661,6 +661,16 @@ "column": "col3", "name": "hash_vin" } + ], + "columns": [ + { + "name": "col3", + "type": "INT16" + }, + { + "name": "coly", + "type": "INT16" + } ] }, "tbl4": {