From 42afc728fe34a63037fb4bedfe0b1ac00bef12dd Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 15 Jan 2024 12:25:59 +0530 Subject: [PATCH] Improve sharded query routing for tuple list (#14892) Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/route_test.go | 43 +++++ go/vt/vtgate/evalengine/cached_size.go | 12 ++ go/vt/vtgate/evalengine/expr_bvar.go | 4 +- go/vt/vtgate/evalengine/expr_tuple_bvar.go | 110 ++++++++++++ .../vtgate/evalengine/expr_tuple_bvar_test.go | 156 +++++++++++++++++ go/vt/vtgate/evalengine/format.go | 12 ++ go/vt/vtgate/evalengine/translate_simplify.go | 8 + go/vt/vtgate/executor_dml_test.go | 13 +- go/vt/vtgate/executor_select_test.go | 69 ++++++++ go/vt/vtgate/executor_test.go | 1 + go/vt/vtgate/planbuilder/operators/route.go | 4 +- .../planbuilder/operators/sharded_routing.go | 63 ++++++- .../planbuilder/testdata/dml_cases.json | 40 +++-- .../planbuilder/testdata/filter_cases.json | 159 ++++++++++++++++++ .../testdata/foreignkey_cases.json | 26 ++- .../testdata/foreignkey_checks_off_cases.json | 10 +- .../testdata/foreignkey_checks_on_cases.json | 26 ++- go/vt/vtgate/testdata/executorVSchema.json | 19 ++- 18 files changed, 727 insertions(+), 48 deletions(-) create mode 100644 go/vt/vtgate/evalengine/expr_tuple_bvar.go create mode 100644 go/vt/vtgate/evalengine/expr_tuple_bvar_test.go diff --git a/go/vt/vtgate/engine/route_test.go b/go/vt/vtgate/engine/route_test.go index 45f8e66c298..0fc5c59bea0 100644 --- a/go/vt/vtgate/engine/route_test.go +++ b/go/vt/vtgate/engine/route_test.go @@ -1716,3 +1716,46 @@ func TestBuildMultiColumnVindexValues(t *testing.T) { }) } } + +// TestSelectTupleMultiCol tests route execution having bind variable with multi column tuple. +func TestSelectTupleMultiCol(t *testing.T) { + vindex, _ := vindexes.CreateVindex("multicol", "", map[string]string{ + "column_count": "2", + "column_vindex": "hash,binary", + }) + + sel := NewRoute( + MultiEqual, + &vindexes.Keyspace{Name: "user", Sharded: true}, + "select 1 from multicol_tbl where (colb, colx, cola) in ::vals", + "select 1 from multicol_tbl where 1 != 1", + ) + sel.Vindex = vindex + sel.Values = []evalengine.Expr{ + &evalengine.TupleBindVariable{Key: "vals", Index: 0}, + &evalengine.TupleBindVariable{Key: "vals", Index: 1}, + } + + v1 := sqltypes.TestTuple(sqltypes.NewInt64(1), sqltypes.NewVarChar("a")) + v2 := sqltypes.TestTuple(sqltypes.NewInt64(4), sqltypes.NewVarChar("b")) + tupleBV := &querypb.BindVariable{ + Type: querypb.Type_TUPLE, + Values: append([]*querypb.Value{sqltypes.ValueToProto(v1)}, sqltypes.ValueToProto(v2)), + } + vc := &loggingVCursor{ + shards: []string{"-20", "20-"}, + } + _, err := sel.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{"vals": tupleBV}, false) + require.NoError(t, err) + vc.ExpectLog(t, []string{ + `ResolveDestinationsMultiCol user [[INT64(1) VARCHAR("a")] [INT64(4) VARCHAR("b")]] Destinations:DestinationKeyspaceID(166b40b461),DestinationKeyspaceID(d2fd886762)`, + `ExecuteMultiShard user.-20: select 1 from multicol_tbl where (colb, colx, cola) in ::vals {vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x011\x950\x01a"} values:{type:TUPLE value:"\x89\x02\x014\x950\x01b"}} false false`, + }) + + vc.Rewind() + _, _ = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{"vals": tupleBV}, false) + vc.ExpectLog(t, []string{ + `ResolveDestinationsMultiCol user [[INT64(1) VARCHAR("a")] [INT64(4) VARCHAR("b")]] Destinations:DestinationKeyspaceID(166b40b461),DestinationKeyspaceID(d2fd886762)`, + `StreamExecuteMulti select 1 from multicol_tbl where (colb, colx, cola) in ::vals user.-20: {vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x011\x950\x01a"} values:{type:TUPLE value:"\x89\x02\x014\x950\x01b"}} `, + }) +} diff --git a/go/vt/vtgate/evalengine/cached_size.go b/go/vt/vtgate/evalengine/cached_size.go index e2cf78b09f1..dfc1c349fc9 100644 --- a/go/vt/vtgate/evalengine/cached_size.go +++ b/go/vt/vtgate/evalengine/cached_size.go @@ -363,6 +363,18 @@ func (cached *OrderByParams) CachedSize(alloc bool) int64 { size += cached.CollationEnv.CachedSize(true) return size } +func (cached *TupleBindVariable) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(48) + } + // field Key string + size += hack.RuntimeAllocSize(int64(len(cached.Key))) + return size +} func (cached *UnaryExpr) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) diff --git a/go/vt/vtgate/evalengine/expr_bvar.go b/go/vt/vtgate/evalengine/expr_bvar.go index dfbaaafffa7..b21ded90189 100644 --- a/go/vt/vtgate/evalengine/expr_bvar.go +++ b/go/vt/vtgate/evalengine/expr_bvar.go @@ -65,7 +65,7 @@ func (bv *BindVariable) eval(env *ExpressionEnv) (eval, error) { switch bvar.Type { case sqltypes.Tuple: if bv.Type != sqltypes.Tuple { - return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "query argument '%s' cannot be a tuple", bv.Key) + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "query argument '%s' must be a tuple (is %s)", bv.Key, bvar.Type) } tuple := make([]eval, 0, len(bvar.Values)) @@ -80,7 +80,7 @@ func (bv *BindVariable) eval(env *ExpressionEnv) (eval, error) { default: if bv.Type == sqltypes.Tuple { - return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "query argument '%s' must be a tuple (is %s)", bv.Key, bvar.Type) + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "query argument '%s' cannot be a tuple", bv.Key) } typ := bvar.Type if bv.typed() { diff --git a/go/vt/vtgate/evalengine/expr_tuple_bvar.go b/go/vt/vtgate/evalengine/expr_tuple_bvar.go new file mode 100644 index 00000000000..b8e506eaff5 --- /dev/null +++ b/go/vt/vtgate/evalengine/expr_tuple_bvar.go @@ -0,0 +1,110 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package evalengine + +import ( + "errors" + + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" +) + +type ( + TupleBindVariable struct { + Key string + + Index int + Type sqltypes.Type + Collation collations.ID + + // dynamicTypeOffset is set when the type of this bind variable cannot be calculated + // at translation time. Since expressions with dynamic types cannot be compiled ahead of time, + // compilation will be delayed until the expression is first executed with the bind variables + // sent by the user. See: UntypedExpr + dynamicTypeOffset int + } +) + +var _ IR = (*TupleBindVariable)(nil) +var _ Expr = (*TupleBindVariable)(nil) + +func (bv *TupleBindVariable) IR() IR { + return bv +} + +func (bv *TupleBindVariable) IsExpr() {} + +// eval implements the expression interface +func (bv *TupleBindVariable) eval(env *ExpressionEnv) (eval, error) { + bvar, err := env.lookupBindVar(bv.Key) + if err != nil { + return nil, err + } + + if bvar.Type != sqltypes.Tuple { + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "query argument '%s' must be a tuple (is %s)", bv.Key, bvar.Type.String()) + } + + tuple := make([]eval, 0, len(bvar.Values)) + for _, value := range bvar.Values { + if value.Type != sqltypes.Tuple { + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "result value must be a tuple (is %s)", value.Type.String()) + } + sValue := sqltypes.ProtoToValue(value) + var evalErr error + idx := 0 + found := false + // looking for a single index on each Tuple Value. + loopErr := sValue.ForEachValue(func(val sqltypes.Value) { + if found || idx != bv.Index { + idx++ + return + } + found = true + e, err := valueToEval(val, typedCoercionCollation(val.Type(), collations.CollationForType(val.Type(), bv.Collation))) + if err != nil { + evalErr = err + return + } + tuple = append(tuple, e) + + }) + if err = errors.Join(loopErr, evalErr); err != nil { + return nil, err + } + if !found { + return nil, vterrors.VT13001("value not found in the bind variable") + } + } + return &evalTuple{t: tuple}, nil +} + +// typeof implements the expression interface +func (bv *TupleBindVariable) typeof(env *ExpressionEnv) (ctype, error) { + _, err := env.lookupBindVar(bv.Key) + if err != nil { + return ctype{}, err + } + + return ctype{Type: sqltypes.Tuple}, nil +} + +func (bv *TupleBindVariable) compile(c *compiler) (ctype, error) { + return ctype{}, c.unsupported(bv) +} diff --git a/go/vt/vtgate/evalengine/expr_tuple_bvar_test.go b/go/vt/vtgate/evalengine/expr_tuple_bvar_test.go new file mode 100644 index 00000000000..cad4d030d1a --- /dev/null +++ b/go/vt/vtgate/evalengine/expr_tuple_bvar_test.go @@ -0,0 +1,156 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package evalengine + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" + querypb "vitess.io/vitess/go/vt/proto/query" +) + +// TestTupleBindVarEval tests TupleBindVariable eval function. +func TestTupleBindVarEval(t *testing.T) { + key := "vals" + c := &TupleBindVariable{ + Key: key, + Index: 1, + } + collation := collations.TypedCollation{ + Coercibility: collations.CoerceCoercible, + Repertoire: collations.RepertoireUnicode, + } + + tcases := []struct { + tName string + bv *querypb.BindVariable + + expEval []eval + expErr string + }{{ + tName: "bind variable not provided", + expErr: "query arguments missing for vals", + }, { + tName: "bind variable provided - wrong type", + bv: sqltypes.Int64BindVariable(1), + expErr: "query argument 'vals' must be a tuple (is INT64)", + }, { + tName: "bind variable provided", + bv: &querypb.BindVariable{ + Type: querypb.Type_TUPLE, + Values: []*querypb.Value{sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(1), sqltypes.NewVarChar("a")))}, + }, + expEval: []eval{newEvalText([]byte("a"), collation)}, + }, { + tName: "bind variable provided - multi values", + bv: &querypb.BindVariable{ + Type: querypb.Type_TUPLE, + Values: []*querypb.Value{ + sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(1), sqltypes.NewVarChar("a"))), + sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(2), sqltypes.NewVarChar("b"))), + sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(3), sqltypes.NewVarChar("c"))), + }, + }, + expEval: []eval{ + newEvalText([]byte("a"), collation), + newEvalText([]byte("b"), collation), + newEvalText([]byte("c"), collation)}, + }} + + for _, tcase := range tcases { + t.Run(tcase.tName, func(t *testing.T) { + env := &ExpressionEnv{ + BindVars: make(map[string]*querypb.BindVariable), + } + if tcase.bv != nil { + env.BindVars[key] = tcase.bv + } + + res, err := c.eval(env) + if tcase.expErr != "" { + require.ErrorContains(t, err, tcase.expErr) + return + } + require.Equal(t, sqltypes.Tuple, res.SQLType()) + resTuple := res.(*evalTuple) + require.Len(t, resTuple.t, len(tcase.expEval)) + for idx, e := range tcase.expEval { + require.Equal(t, e, resTuple.t[idx]) + } + }) + } +} + +// TestTupleBindVarTypeOf tests TupleBindVariable typeOf function. +func TestTupleBindVarTypeOf(t *testing.T) { + key := "vals" + c := &TupleBindVariable{ + Key: key, + Index: 1, + } + + tcases := []struct { + tName string + bv *querypb.BindVariable + + expErr string + }{{ + tName: "bind variable not provided", + expErr: "query arguments missing for vals", + }, { + // typeOf does not evaluate the bind variable value + tName: "bind variable provided - wrong type", + bv: sqltypes.Int64BindVariable(1), + }, { + tName: "bind variable provided", + bv: &querypb.BindVariable{ + Type: querypb.Type_TUPLE, + Values: []*querypb.Value{sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(1), sqltypes.NewVarChar("a")))}, + }, + }, { + tName: "bind variable provided - multi values", + bv: &querypb.BindVariable{ + Type: querypb.Type_TUPLE, + Values: []*querypb.Value{ + sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(1), sqltypes.NewVarChar("a"))), + sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(2), sqltypes.NewVarChar("b"))), + sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(3), sqltypes.NewVarChar("c"))), + }, + }, + }} + + for _, tcase := range tcases { + t.Run(tcase.tName, func(t *testing.T) { + env := &ExpressionEnv{ + BindVars: make(map[string]*querypb.BindVariable), + } + if tcase.bv != nil { + env.BindVars[key] = tcase.bv + } + + res, err := c.typeof(env) + if tcase.expErr != "" { + require.ErrorContains(t, err, tcase.expErr) + return + } + require.Equal(t, sqltypes.Tuple, res.Type) + }) + } +} diff --git a/go/vt/vtgate/evalengine/format.go b/go/vt/vtgate/evalengine/format.go index ee6b66b192a..dddf538b37c 100644 --- a/go/vt/vtgate/evalengine/format.go +++ b/go/vt/vtgate/evalengine/format.go @@ -152,6 +152,18 @@ func (bv *BindVariable) format(buf *sqlparser.TrackedBuffer) { } } +func (bv *TupleBindVariable) Format(buf *sqlparser.TrackedBuffer) { + bv.format(buf) +} + +func (bv *TupleBindVariable) FormatFast(buf *sqlparser.TrackedBuffer) { + bv.format(buf) +} + +func (bv *TupleBindVariable) format(buf *sqlparser.TrackedBuffer) { + buf.WriteString(fmt.Sprintf("%s:%d", bv.Key, bv.Index)) +} + func (c *Column) Format(buf *sqlparser.TrackedBuffer) { c.format(buf) } diff --git a/go/vt/vtgate/evalengine/translate_simplify.go b/go/vt/vtgate/evalengine/translate_simplify.go index 2537ad20b2b..6af7f7646a0 100644 --- a/go/vt/vtgate/evalengine/translate_simplify.go +++ b/go/vt/vtgate/evalengine/translate_simplify.go @@ -26,6 +26,10 @@ func (expr *BindVariable) constant() bool { return false } +func (expr *TupleBindVariable) constant() bool { + return false +} + func (expr *Column) constant() bool { return false } @@ -55,6 +59,10 @@ func (expr *BindVariable) simplify(_ *ExpressionEnv) error { return nil } +func (expr *TupleBindVariable) simplify(_ *ExpressionEnv) error { + return nil +} + func (expr *Column) simplify(_ *ExpressionEnv) error { return nil } diff --git a/go/vt/vtgate/executor_dml_test.go b/go/vt/vtgate/executor_dml_test.go index 3d48e96dabc..b427e198e5e 100644 --- a/go/vt/vtgate/executor_dml_test.go +++ b/go/vt/vtgate/executor_dml_test.go @@ -3065,16 +3065,9 @@ func TestDeleteMulti(t *testing.T) { testQueryLog(t, executor, logChan, "MarkSavepoint", "SAVEPOINT", "savepoint s1", 8) testQueryLog(t, executor, logChan, "VindexDelete", "DELETE", "delete from name_user_map where `name` = :name and user_id = :user_id", 1) - testQueryLog(t, executor, logChan, "VindexDelete", "DELETE", "delete from name_user_map where `name` = :name and user_id = :user_id", 1) - testQueryLog(t, executor, logChan, "VindexDelete", "DELETE", "delete from name_user_map where `name` = :name and user_id = :user_id", 1) - testQueryLog(t, executor, logChan, "VindexDelete", "DELETE", "delete from name_user_map where `name` = :name and user_id = :user_id", 1) - testQueryLog(t, executor, logChan, "VindexDelete", "DELETE", "delete from name_user_map where `name` = :name and user_id = :user_id", 1) - testQueryLog(t, executor, logChan, "VindexDelete", "DELETE", "delete from name_user_map where `name` = :name and user_id = :user_id", 1) - testQueryLog(t, executor, logChan, "VindexDelete", "DELETE", "delete from name_user_map where `name` = :name and user_id = :user_id", 1) - testQueryLog(t, executor, logChan, "VindexDelete", "DELETE", "delete from name_user_map where `name` = :name and user_id = :user_id", 1) // select `user`.id, `user`.col from `user` - 8 shard // select 1 from music where music.user_id = 1 and music.col = :user_col - 8 shards - // select Id, `name` from `user` where (`user`.id) in ::dm_vals for update - 8 shards - // delete from `user` where (`user`.id) in ::dm_vals - 8 shards - testQueryLog(t, executor, logChan, "TestExecute", "DELETE", "delete `user` from `user` join music on `user`.col = music.col where music.user_id = 1", 32) + // select Id, `name` from `user` where (`user`.id) in ::dm_vals for update - 1 shard + // delete from `user` where (`user`.id) in ::dm_vals - 1 shard + testQueryLog(t, executor, logChan, "TestExecute", "DELETE", "delete `user` from `user` join music on `user`.col = music.col where music.user_id = 1", 18) } diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index af40c0e17b3..43988257e03 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -1552,6 +1552,75 @@ func TestStreamSelectIN(t *testing.T) { utils.MustMatch(t, wantQueries, sbclookup.Queries) } +// TestSelectListArg tests list arg filter with select query +func TestSelectListArg(t *testing.T) { + executor, sbc1, sbc2, _, ctx := createExecutorEnv(t) + session := &vtgatepb.Session{ + TargetString: "@primary", + } + + tupleBV := &querypb.BindVariable{ + Type: querypb.Type_TUPLE, + Values: []*querypb.Value{sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(1), sqltypes.NewVarChar("a")))}, + } + bvMap := map[string]*querypb.BindVariable{"vals": tupleBV} + _, err := executorExec(ctx, executor, session, "select id from user where (id, col) in ::vals", bvMap) + require.NoError(t, err) + wantQueries := []*querypb.BoundQuery{{ + Sql: "select id from `user` where (id, col) in ::vals", + BindVariables: bvMap, + }} + utils.MustMatch(t, wantQueries, sbc1.Queries) + assert.Nil(t, sbc2.Queries, "sbc2.Queries: %+v, want nil", sbc2.Queries) + + sbc1.Queries = nil + // get c0-e0 sandbox connection. + tbh, err := executor.scatterConn.gateway.hc.GetTabletHealthByAlias(&topodatapb.TabletAlias{ + Cell: "aa", + Uid: 7, + }) + require.NoError(t, err) + sbc := tbh.Conn.(*sandboxconn.SandboxConn) + sbc.Queries = nil + + _, err = executorExec(ctx, executor, session, "select id from multicol_tbl where (cola, colb) in ::vals", bvMap) + require.NoError(t, err) + + wantQueries = []*querypb.BoundQuery{{ + Sql: "select id from multicol_tbl where (cola, colb) in ::vals", + BindVariables: bvMap, + }} + utils.MustMatch(t, wantQueries, sbc.Queries) + assert.Nil(t, sbc1.Queries, "sbc1.Queries: %+v, want nil", sbc2.Queries) + assert.Nil(t, sbc2.Queries, "sbc2.Queries: %+v, want nil", sbc2.Queries) + + tupleBV.Values[0] = sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewInt64(1), sqltypes.NewInt64(42), sqltypes.NewVarChar("a"))) + sbc.Queries = nil + _, err = executorExec(ctx, executor, session, "select id from multicol_tbl where (cola, colx, colb) in ::vals", bvMap) + require.NoError(t, err) + + wantQueries = []*querypb.BoundQuery{{ + Sql: "select id from multicol_tbl where (cola, colx, colb) in ::vals", + BindVariables: bvMap, + }} + utils.MustMatch(t, wantQueries, sbc.Queries) + assert.Nil(t, sbc1.Queries, "sbc1.Queries: %+v, want nil", sbc2.Queries) + assert.Nil(t, sbc2.Queries, "sbc2.Queries: %+v, want nil", sbc2.Queries) + + tupleBV.Values[0] = sqltypes.ValueToProto(sqltypes.TestTuple(sqltypes.NewVarChar("a"), sqltypes.NewInt64(42), sqltypes.NewInt64(1))) + sbc.Queries = nil + _, err = executorExec(ctx, executor, session, "select id from multicol_tbl where (colb, colx, cola) in ::vals", bvMap) + require.NoError(t, err) + + wantQueries = []*querypb.BoundQuery{{ + Sql: "select id from multicol_tbl where (colb, colx, cola) in ::vals", + BindVariables: bvMap, + }} + utils.MustMatch(t, wantQueries, sbc.Queries) + assert.Nil(t, sbc1.Queries, "sbc1.Queries: %+v, want nil", sbc2.Queries) + assert.Nil(t, sbc2.Queries, "sbc2.Queries: %+v, want nil", sbc2.Queries) +} + func createExecutor(ctx context.Context, serv *sandboxTopo, cell string, resolver *Resolver) *Executor { queryLogger := streamlog.New[*logstats.LogStats]("VTGate", queryLogBufferSize) plans := DefaultPlanCache() diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index b0ab47ebed3..bbb92513927 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -889,6 +889,7 @@ func TestExecutorShow(t *testing.T) { buildVarCharRow("TestExecutor", "keyspace_id", "numeric", "", ""), buildVarCharRow("TestExecutor", "krcol_unique_vdx", "keyrange_lookuper_unique", "", ""), buildVarCharRow("TestExecutor", "krcol_vdx", "keyrange_lookuper", "", ""), + buildVarCharRow("TestExecutor", "multicol_vdx", "multicol", "column_count=2; column_vindex=xxhash,binary", ""), buildVarCharRow("TestExecutor", "music_user_map", "lookup_hash_unique", "from=music_id; table=music_user_map; to=user_id", "music"), buildVarCharRow("TestExecutor", "name_lastname_keyspace_id_map", "lookup", "from=name,lastname; table=name_lastname_keyspace_id_map; to=keyspace_id", "user2"), buildVarCharRow("TestExecutor", "name_user_map", "lookup_hash", "from=name; table=name_user_map; to=user_id", "user"), diff --git a/go/vt/vtgate/planbuilder/operators/route.go b/go/vt/vtgate/planbuilder/operators/route.go index 952e455abb0..25612b42ff0 100644 --- a/go/vt/vtgate/planbuilder/operators/route.go +++ b/go/vt/vtgate/planbuilder/operators/route.go @@ -261,7 +261,9 @@ func (option *VindexOption) updateWithNewColumn( opcode func(*vindexes.ColumnVindex) engine.Opcode, ) bool { option.ColsSeen[colLoweredName] = true - option.ValueExprs = append(option.ValueExprs, valueExpr) + if valueExpr != nil { + option.ValueExprs = append(option.ValueExprs, valueExpr) + } option.Values[indexOfCol] = value option.Predicates[indexOfCol] = node option.Ready = len(option.ColsSeen) == len(colVindex.Columns) diff --git a/go/vt/vtgate/planbuilder/operators/sharded_routing.go b/go/vt/vtgate/planbuilder/operators/sharded_routing.go index 0bde10946fb..cb333f34cee 100644 --- a/go/vt/vtgate/planbuilder/operators/sharded_routing.go +++ b/go/vt/vtgate/planbuilder/operators/sharded_routing.go @@ -259,13 +259,13 @@ func (tr *ShardedRouting) planInOp(ctx *plancontext.PlanningContext, cmp *sqlpar opcode := func(*vindexes.ColumnVindex) engine.Opcode { return engine.IN } return tr.haveMatchingVindex(ctx, cmp, vdValue, left, value, opcode, justTheVindex) case sqlparser.ValTuple: - right, rightIsValTuple := cmp.Right.(sqlparser.ValTuple) - if !rightIsValTuple { - return false + switch right := cmp.Right.(type) { + case sqlparser.ValTuple: + return tr.planCompositeInOpRecursive(ctx, cmp, left, right, nil) + case sqlparser.ListArg: + return tr.planCompositeInOpArg(ctx, cmp, left, right) } - return tr.planCompositeInOpRecursive(ctx, cmp, left, right, nil) } - return false } @@ -348,7 +348,6 @@ func (tr *ShardedRouting) haveMatchingVindex( switch v.ColVindex.Vindex.(type) { case vindexes.SingleColumn: newVindexFound = tr.processSingleColumnVindex(node, valueExpr, column, value, opcode, vfunc, v, newVindexFound) - case vindexes.MultiColumn: newVindexFound = tr.processMultiColumnVindex(node, valueExpr, column, value, opcode, vfunc, v, newVindexFound) } @@ -378,15 +377,19 @@ func (tr *ShardedRouting) processSingleColumnVindex( return newVindexFound } - vindexPlusPredicates.Options = append(vindexPlusPredicates.Options, &VindexOption{ + vo := &VindexOption{ Values: []evalengine.Expr{value}, - ValueExprs: []sqlparser.Expr{valueExpr}, Predicates: []sqlparser.Expr{node}, OpCode: routeOpcode, FoundVindex: vindex, Cost: costFor(vindexPlusPredicates.ColVindex, routeOpcode), Ready: true, - }) + } + if valueExpr != nil { + vo.ValueExprs = []sqlparser.Expr{valueExpr} + } + vindexPlusPredicates.Options = append(vindexPlusPredicates.Options, vo) + return true } @@ -515,6 +518,40 @@ func (tr *ShardedRouting) planCompositeInOpRecursive( return foundVindex } +func (tr *ShardedRouting) planCompositeInOpArg( + ctx *plancontext.PlanningContext, + cmp *sqlparser.ComparisonExpr, + left sqlparser.ValTuple, + right sqlparser.ListArg, +) bool { + foundVindex := false + for idx, expr := range left { + col, ok := expr.(*sqlparser.ColName) + if !ok { + continue + } + + // check if left col is a vindex + if !tr.hasVindex(col) { + continue + } + + value := &evalengine.TupleBindVariable{ + Key: right.String(), + Index: idx, + } + if typ, found := ctx.SemTable.TypeForExpr(col); found { + value.Type = typ.Type() + value.Collation = typ.Collation() + } + + opcode := func(*vindexes.ColumnVindex) engine.Opcode { return engine.MultiEqual } + newVindex := tr.haveMatchingVindex(ctx, cmp, nil, col, value, opcode, justTheVindex) + foundVindex = newVindex || foundVindex + } + return foundVindex +} + func (tr *ShardedRouting) hasVindex(column *sqlparser.ColName) bool { for _, v := range tr.VindexPreds { for _, col := range v.ColVindex.Columns { @@ -548,6 +585,14 @@ func (tr *ShardedRouting) extraInfo() string { ) } + if len(tr.Selected.ValueExprs) == 0 { + return fmt.Sprintf( + "Vindex[%s] Seen:[%s]", + tr.Selected.FoundVindex.String(), + sqlparser.String(sqlparser.AndExpressions(tr.SeenPredicates...)), + ) + } + return fmt.Sprintf( "Vindex[%s] Values[%s] Seen:[%s]", tr.Selected.FoundVindex.String(), diff --git a/go/vt/vtgate/planbuilder/testdata/dml_cases.json b/go/vt/vtgate/planbuilder/testdata/dml_cases.json index 36bd0943bce..2e8053c2e78 100644 --- a/go/vt/vtgate/planbuilder/testdata/dml_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/dml_cases.json @@ -5097,7 +5097,7 @@ }, { "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "user", "Sharded": true @@ -5107,7 +5107,11 @@ "KsidVindex": "user_index", "OwnedVindexQuery": "select `user`.Id, `user`.`Name`, `user`.Costly from `user` where (`user`.id) in ::dm_vals for update", "Query": "delete from `user` where (`user`.id) in ::dm_vals", - "Table": "user" + "Table": "user", + "Values": [ + "dm_vals:0" + ], + "Vindex": "user_index" } ] }, @@ -5162,7 +5166,7 @@ }, { "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "user", "Sharded": true @@ -5172,7 +5176,11 @@ "KsidVindex": "user_index", "OwnedVindexQuery": "select u.Id, u.`Name`, u.Costly from `user` as u where (u.id) in ::dm_vals for update", "Query": "delete from `user` as u where (u.id) in ::dm_vals", - "Table": "user" + "Table": "user", + "Values": [ + "dm_vals:0" + ], + "Vindex": "user_index" } ] }, @@ -5227,7 +5235,7 @@ }, { "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "user", "Sharded": true @@ -5237,7 +5245,11 @@ "KsidVindex": "user_index", "OwnedVindexQuery": "select u.Id, u.`Name`, u.Costly from `user` as u where (u.id) in ::dm_vals for update", "Query": "delete from `user` as u where (u.id) in ::dm_vals", - "Table": "user" + "Table": "user", + "Values": [ + "dm_vals:0" + ], + "Vindex": "user_index" } ] }, @@ -5318,7 +5330,7 @@ }, { "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "user", "Sharded": true @@ -5328,7 +5340,11 @@ "KsidVindex": "user_index", "OwnedVindexQuery": "select u.Id, u.`Name`, u.Costly from `user` as u where (u.id) in ::dm_vals for update", "Query": "delete from `user` as u where (u.id) in ::dm_vals", - "Table": "user" + "Table": "user", + "Values": [ + "dm_vals:0" + ], + "Vindex": "user_index" } ] }, @@ -5384,7 +5400,7 @@ }, { "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "user", "Sharded": true @@ -5394,7 +5410,11 @@ "KsidVindex": "user_index", "OwnedVindexQuery": "select m.user_id, m.id from music as m where (m.id) in ::dm_vals for update", "Query": "delete from music as m where (m.id) in ::dm_vals", - "Table": "music" + "Table": "music", + "Values": [ + "dm_vals:0" + ], + "Vindex": "music_user_map" } ] }, diff --git a/go/vt/vtgate/planbuilder/testdata/filter_cases.json b/go/vt/vtgate/planbuilder/testdata/filter_cases.json index dc8614a8ba7..c71903e0b6c 100644 --- a/go/vt/vtgate/planbuilder/testdata/filter_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/filter_cases.json @@ -4402,5 +4402,164 @@ "user.user_extra" ] } + }, + { + "comment": "list args: single column vindex", + "query": "select 1 from user where (id, col) in ::vals", + "plan": { + "QueryType": "SELECT", + "Original": "select 1 from user where (id, col) in ::vals", + "Instructions": { + "OperatorType": "Route", + "Variant": "MultiEqual", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from `user` where 1 != 1", + "Query": "select 1 from `user` where (id, col) in ::vals", + "Table": "`user`", + "Values": [ + "vals:0" + ], + "Vindex": "user_index" + }, + "TablesUsed": [ + "user.user" + ] + } + }, + { + "comment": "list args: single column vindex on non-zero offset", + "query": "select 1 from user where (col, id) in ::vals", + "plan": { + "QueryType": "SELECT", + "Original": "select 1 from user where (col, id) in ::vals", + "Instructions": { + "OperatorType": "Route", + "Variant": "MultiEqual", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from `user` where 1 != 1", + "Query": "select 1 from `user` where (col, id) in ::vals", + "Table": "`user`", + "Values": [ + "vals:1" + ], + "Vindex": "user_index" + }, + "TablesUsed": [ + "user.user" + ] + } + }, + { + "comment": "list args: multi column vindex", + "query": "select 1 from multicol_tbl where (cola, colb) in ::vals", + "plan": { + "QueryType": "SELECT", + "Original": "select 1 from multicol_tbl where (cola, colb) in ::vals", + "Instructions": { + "OperatorType": "Route", + "Variant": "MultiEqual", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from multicol_tbl where 1 != 1", + "Query": "select 1 from multicol_tbl where (cola, colb) in ::vals", + "Table": "multicol_tbl", + "Values": [ + "vals:0", + "vals:1" + ], + "Vindex": "multicolIdx" + }, + "TablesUsed": [ + "user.multicol_tbl" + ] + } + }, + { + "comment": "list args: multi column vindex - subshard", + "query": "select 1 from multicol_tbl where (cola) in ::vals", + "plan": { + "QueryType": "SELECT", + "Original": "select 1 from multicol_tbl where (cola) in ::vals", + "Instructions": { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from multicol_tbl where 1 != 1", + "Query": "select 1 from multicol_tbl where cola in ::__vals0", + "Table": "multicol_tbl", + "Values": [ + "::vals" + ], + "Vindex": "multicolIdx" + }, + "TablesUsed": [ + "user.multicol_tbl" + ] + } + }, + { + "comment": "list args: multi column vindex - more columns", + "query": "select 1 from multicol_tbl where (cola, colx, colb) in ::vals", + "plan": { + "QueryType": "SELECT", + "Original": "select 1 from multicol_tbl where (cola, colx, colb) in ::vals", + "Instructions": { + "OperatorType": "Route", + "Variant": "MultiEqual", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from multicol_tbl where 1 != 1", + "Query": "select 1 from multicol_tbl where (cola, colx, colb) in ::vals", + "Table": "multicol_tbl", + "Values": [ + "vals:0", + "vals:2" + ], + "Vindex": "multicolIdx" + }, + "TablesUsed": [ + "user.multicol_tbl" + ] + } + }, + { + "comment": "list args: multi column vindex - columns rearranged", + "query": "select 1 from multicol_tbl where (colb, colx, cola) in ::vals", + "plan": { + "QueryType": "SELECT", + "Original": "select 1 from multicol_tbl where (colb, colx, cola) in ::vals", + "Instructions": { + "OperatorType": "Route", + "Variant": "MultiEqual", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from multicol_tbl where 1 != 1", + "Query": "select 1 from multicol_tbl where (colb, colx, cola) in ::vals", + "Table": "multicol_tbl", + "Values": [ + "vals:2", + "vals:0" + ], + "Vindex": "multicolIdx" + }, + "TablesUsed": [ + "user.multicol_tbl" + ] + } } ] diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 15ac4acf872..d4d582af2aa 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -94,7 +94,7 @@ { "InputName": "CascadeChild-1", "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "sharded_fk_allow", "Sharded": true @@ -109,7 +109,13 @@ 4 ], "Query": "delete from multicol_tbl2 where (colb, cola, x, colc, y) in ::fkc_vals", - "Table": "multicol_tbl2" + "Table": "multicol_tbl2", + "Values": [ + "fkc_vals:1", + "fkc_vals:0", + "fkc_vals:3" + ], + "Vindex": "multicolIdx" }, { "InputName": "Parent", @@ -161,7 +167,7 @@ { "InputName": "CascadeChild-1", "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "sharded_fk_allow", "Sharded": true @@ -172,7 +178,11 @@ 0 ], "Query": "delete from tbl4 where (col4) in ::fkc_vals", - "Table": "tbl4" + "Table": "tbl4", + "Values": [ + "fkc_vals:0" + ], + "Vindex": "hash_vin" }, { "InputName": "CascadeChild-2", @@ -2175,7 +2185,7 @@ { "InputName": "CascadeChild-1", "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "sharded_fk_allow", "Sharded": true @@ -2186,7 +2196,11 @@ 0 ], "Query": "delete from tbl4 where (col4) in ::fkc_vals", - "Table": "tbl4" + "Table": "tbl4", + "Values": [ + "fkc_vals:0" + ], + "Vindex": "hash_vin" }, { "InputName": "CascadeChild-2", diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_off_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_off_cases.json index baa82c89dfd..31ee3e7c554 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_off_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_off_cases.json @@ -123,7 +123,7 @@ { "InputName": "CascadeChild-1", "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "sharded_fk_allow", "Sharded": true @@ -138,7 +138,13 @@ 4 ], "Query": "delete /*+ SET_VAR(foreign_key_checks=ON) */ from multicol_tbl2 where (colb, cola, x, colc, y) in ::fkc_vals", - "Table": "multicol_tbl2" + "Table": "multicol_tbl2", + "Values": [ + "fkc_vals:1", + "fkc_vals:0", + "fkc_vals:3" + ], + "Vindex": "multicolIdx" }, { "InputName": "Parent", diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_on_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_on_cases.json index 24775f5076f..76a71986842 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_on_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_on_cases.json @@ -94,7 +94,7 @@ { "InputName": "CascadeChild-1", "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "sharded_fk_allow", "Sharded": true @@ -109,7 +109,13 @@ 4 ], "Query": "delete /*+ SET_VAR(foreign_key_checks=ON) */ from multicol_tbl2 where (colb, cola, x, colc, y) in ::fkc_vals", - "Table": "multicol_tbl2" + "Table": "multicol_tbl2", + "Values": [ + "fkc_vals:1", + "fkc_vals:0", + "fkc_vals:3" + ], + "Vindex": "multicolIdx" }, { "InputName": "Parent", @@ -161,7 +167,7 @@ { "InputName": "CascadeChild-1", "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "sharded_fk_allow", "Sharded": true @@ -172,7 +178,11 @@ 0 ], "Query": "delete /*+ SET_VAR(foreign_key_checks=ON) */ from tbl4 where (col4) in ::fkc_vals", - "Table": "tbl4" + "Table": "tbl4", + "Values": [ + "fkc_vals:0" + ], + "Vindex": "hash_vin" }, { "InputName": "CascadeChild-2", @@ -2175,7 +2185,7 @@ { "InputName": "CascadeChild-1", "OperatorType": "Delete", - "Variant": "Scatter", + "Variant": "MultiEqual", "Keyspace": { "Name": "sharded_fk_allow", "Sharded": true @@ -2186,7 +2196,11 @@ 0 ], "Query": "delete /*+ SET_VAR(foreign_key_checks=ON) */ from tbl4 where (col4) in ::fkc_vals", - "Table": "tbl4" + "Table": "tbl4", + "Values": [ + "fkc_vals:0" + ], + "Vindex": "hash_vin" }, { "InputName": "CascadeChild-2", diff --git a/go/vt/vtgate/testdata/executorVSchema.json b/go/vt/vtgate/testdata/executorVSchema.json index da12a3b9946..e353c082374 100644 --- a/go/vt/vtgate/testdata/executorVSchema.json +++ b/go/vt/vtgate/testdata/executorVSchema.json @@ -132,6 +132,13 @@ }, "cfc": { "type": "cfc" + }, + "multicol_vdx": { + "type": "multicol", + "params": { + "column_count": "2", + "column_vindex": "xxhash,binary" + } } }, "tables": { @@ -346,6 +353,14 @@ "zip_detail": { "type": "reference", "source": "TestUnsharded.zip_detail" - } - } + }, + "multicol_tbl": { + "column_vindexes": [ + { + "columns": ["cola", "colb"], + "name": "multicol_vdx" + } + ] + } + } }