Skip to content

Commit

Permalink
[release-18.0] Add support for MultiEqual opcode for lookup vindexe…
Browse files Browse the repository at this point in the history 
…s. (#16975) (#17038)

Signed-off-by: Arthur Schreiber <[email protected]>
Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com>
Co-authored-by: Arthur Schreiber <[email protected]>
  • Loading branch information
@vitess-bot @arthurschreiber
vitess-bot[bot] and arthurschreiber authored Oct 24, 2024
1 parent f2c9e0a commit c58abd9
Show file tree
Hide file tree
Showing 5 changed files with 282 additions and 2 deletions.
16 changes: 16 additions & 0 deletions go/test/endtoend/vtgate/lookup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,22 @@ func TestConsistentLookupUpdate(t *testing.T) {
require.Empty(t, qr.Rows)
}

func TestSelectMultiEqualLookup(t *testing.T) {
conn, closer := start(t)
defer closer()

utils.Exec(t, conn, "insert into t10 (id, sharding_key, col1) values (1, 1, 'bar'), (2, 1, 'bar'), (3, 1, 'bar'), (4, 2, 'bar'), (5, 2, 'bar')")

for _, workload := range []string{"oltp", "olap"} {
t.Run(workload, func(t *testing.T) {
utils.Exec(t, conn, "set workload = "+workload)

utils.AssertMatches(t, conn, "select id from t10 WHERE (col1, id) IN (('bar', 1), ('baz', 2), ('qux', 3), ('barbar', 4))", "[[INT64(1)]]")
utils.AssertMatches(t, conn, "select id from t10 WHERE (col1 = 'bar' AND id = 1) OR (col1 = 'baz' AND id = 2) OR (col1 = 'qux' AND id = 3) OR (col1 = 'barbar' AND id = 4)", "[[INT64(1)]]")
})
}
}

func TestSelectNullLookup(t *testing.T) {
conn, closer := start(t)
defer closer()
Expand Down
2 changes: 1 addition & 1 deletion go/test/endtoend/vtgate/schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,4 +164,4 @@ create table t11
col2 int,
col3 int,
primary key (id)
) Engine = InnoDB;
) Engine = InnoDB;
2 changes: 1 addition & 1 deletion go/vt/vtgate/engine/vindex_lookup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ func (vr *VindexLookup) generateIds(ctx context.Context, vcursor VCursor, bindVa
switch vr.Opcode {
case Equal, EqualUnique:
return []sqltypes.Value{value.Value(vcursor.ConnCollation())}, nil
case IN:
case IN, MultiEqual:
return value.TupleValues(), nil
}
return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "opcode %s not supported for VindexLookup", vr.Opcode.String())
Expand Down
191 changes: 191 additions & 0 deletions go/vt/vtgate/executor_select_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2874,6 +2874,197 @@ func TestSubQueryAndQueryWithLimit(t *testing.T) {
assert.Equal(t, `type:INT64 value:"100"`, sbc2.Queries[1].BindVariables["__upper_limit"].String())
}

func TestSelectUsingLookupColumn(t *testing.T) {
t.Run("using multi value tuple", func(t *testing.T) {
ctx := utils.LeakCheckContext(t)

// Special setup: Don't use createExecutorEnv.
cell := "aa"
hc := discovery.NewFakeHealthCheck(nil)

u := createSandbox(KsTestUnsharded)
s := createSandbox(KsTestSharded)

s.VSchema = executorVSchema
u.VSchema = unshardedVSchema

serv := newSandboxForCells(ctx, []string{cell})
resolver := newTestResolver(ctx, hc, serv, cell)

shards := []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"}
sbcs := []*sandboxconn.SandboxConn{}
for _, shard := range shards {
sbcs = append(sbcs, hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil))
}

sbclookup := hc.AddTestTablet(cell, "0", 1, KsTestUnsharded, "0", topodatapb.TabletType_PRIMARY, true, 1, nil)

executor := createExecutor(ctx, serv, cell, resolver)
defer executor.Close()
logChan := executor.queryLogger.Subscribe("Test")
defer executor.queryLogger.Unsubscribe(logChan)

// Only lookup results on shard `40-60` (`sbc[2]`)
sbclookup.SetResults([]*sqltypes.Result{{
Fields: []*querypb.Field{
{Name: "lu_col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)},
{Name: "keyspace_id", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)},
},
Rows: [][]sqltypes.Value{{
sqltypes.NewInt32(2),
sqltypes.MakeTrusted(sqltypes.VarBinary, []byte("\x45")),
}},
}})

sbcs[2].SetResults([]*sqltypes.Result{{
Fields: []*querypb.Field{
{Name: "nv_lu_col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)},
{Name: "other", Type: sqltypes.VarChar, Charset: collations.CollationUtf8mb4ID},
},
Rows: [][]sqltypes.Value{{
sqltypes.NewInt32(2),
sqltypes.NewVarChar("baz"),
}},
}})

result, err := exec(executor, NewSafeSession(&vtgatepb.Session{
TargetString: KsTestSharded,
}), "select nv_lu_col, other from t2_lookup WHERE (nv_lu_col, other) IN ((1, 'bar'), (2, 'baz'), (3, 'qux'), (4, 'brz'), (5, 'brz'))")

require.NoError(t, err)

require.Len(t, sbclookup.Queries, 1)
require.Len(t, sbcs[0].Queries, 0)
require.Len(t, sbcs[1].Queries, 0)
require.Len(t, sbcs[2].Queries, 1)
require.Len(t, sbcs[3].Queries, 0)
require.Len(t, sbcs[4].Queries, 0)
require.Len(t, sbcs[5].Queries, 0)
require.Len(t, sbcs[6].Queries, 0)
require.Len(t, sbcs[7].Queries, 0)

require.Equal(t, []*querypb.BoundQuery{{
Sql: "select nv_lu_col, other from t2_lookup where (nv_lu_col, other) in ((1, 'bar'), (2, 'baz'), (3, 'qux'), (4, 'brz'), (5, 'brz'))",
BindVariables: map[string]*querypb.BindVariable{},
}}, sbcs[2].Queries)

wantResult := &sqltypes.Result{
Fields: []*querypb.Field{
{Name: "nv_lu_col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)},
{Name: "other", Type: sqltypes.VarChar, Charset: collations.CollationUtf8mb4ID},
},
Rows: [][]sqltypes.Value{{
sqltypes.NewInt32(2),
sqltypes.NewVarChar("baz"),
}},
}
require.Equal(t, wantResult, result)
})

t.Run("using disjunction of conjunctions", func(t *testing.T) {
ctx := utils.LeakCheckContext(t)

// Special setup: Don't use createExecutorEnv.
cell := "aa"
hc := discovery.NewFakeHealthCheck(nil)

u := createSandbox(KsTestUnsharded)
s := createSandbox(KsTestSharded)

s.VSchema = executorVSchema
u.VSchema = unshardedVSchema

serv := newSandboxForCells(ctx, []string{cell})
resolver := newTestResolver(ctx, hc, serv, cell)

shards := []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"}
sbcs := []*sandboxconn.SandboxConn{}
for _, shard := range shards {
sbcs = append(sbcs, hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil))
}

sbclookup := hc.AddTestTablet(cell, "0", 1, KsTestUnsharded, "0", topodatapb.TabletType_PRIMARY, true, 1, nil)

executor := createExecutor(ctx, serv, cell, resolver)
defer executor.Close()
logChan := executor.queryLogger.Subscribe("Test")
defer executor.queryLogger.Unsubscribe(logChan)

// Only lookup results on shard `40-60` (`sbc[2]`)
sbclookup.SetResults([]*sqltypes.Result{{
Fields: []*querypb.Field{
{Name: "lu_col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)},
{Name: "keyspace_id", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)},
},
Rows: [][]sqltypes.Value{{
sqltypes.NewInt32(2),
sqltypes.MakeTrusted(sqltypes.VarBinary, []byte("\x45")),
}},
}})

emptyResult := []*sqltypes.Result{{
Fields: []*querypb.Field{
{Name: "nv_lu_col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)},
{Name: "other", Type: sqltypes.VarChar, Charset: collations.CollationUtf8mb4ID},
},
Rows: [][]sqltypes.Value{},
}}

sbcs[0].SetResults(emptyResult)
sbcs[1].SetResults(emptyResult)
sbcs[2].SetResults([]*sqltypes.Result{{
Fields: []*querypb.Field{
{Name: "nv_lu_col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)},
{Name: "other", Type: sqltypes.VarChar, Charset: collations.CollationUtf8mb4ID},
},
Rows: [][]sqltypes.Value{{
sqltypes.NewInt32(2),
sqltypes.NewVarChar("baz"),
}},
}})
sbcs[3].SetResults(emptyResult)
sbcs[4].SetResults(emptyResult)
sbcs[5].SetResults(emptyResult)
sbcs[6].SetResults(emptyResult)
sbcs[7].SetResults(emptyResult)

result, err := exec(executor, NewSafeSession(&vtgatepb.Session{
TargetString: KsTestSharded,
}), "select nv_lu_col, other from t2_lookup WHERE (nv_lu_col = 1 AND other = 'bar') OR (nv_lu_col = 2 AND other = 'baz') OR (nv_lu_col = 3 AND other = 'qux') OR (nv_lu_col = 4 AND other = 'brz') OR (nv_lu_col = 5 AND other = 'brz')")

require.NoError(t, err)

// We end up doing a scatter query here, so no queries are sent to the lookup table
require.Len(t, sbclookup.Queries, 0)
require.Len(t, sbcs[0].Queries, 1)
require.Len(t, sbcs[1].Queries, 1)
require.Len(t, sbcs[2].Queries, 1)
require.Len(t, sbcs[3].Queries, 1)
require.Len(t, sbcs[4].Queries, 1)
require.Len(t, sbcs[5].Queries, 1)
require.Len(t, sbcs[6].Queries, 1)
require.Len(t, sbcs[7].Queries, 1)

for _, sbc := range sbcs {
require.Equal(t, []*querypb.BoundQuery{{
Sql: "select nv_lu_col, other from t2_lookup where nv_lu_col = 1 and other = 'bar' or nv_lu_col = 2 and other = 'baz' or nv_lu_col = 3 and other = 'qux' or nv_lu_col = 4 and other = 'brz' or nv_lu_col = 5 and other = 'brz'",
BindVariables: map[string]*querypb.BindVariable{},
}}, sbc.Queries)
}
wantResult := &sqltypes.Result{
Fields: []*querypb.Field{
{Name: "nv_lu_col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)},
{Name: "other", Type: sqltypes.VarChar, Charset: collations.CollationUtf8mb4ID},
},
Rows: [][]sqltypes.Value{{
sqltypes.NewInt32(2),
sqltypes.NewVarChar("baz"),
}},
}
require.Equal(t, wantResult, result)
})
}

func TestCrossShardSubqueryStream(t *testing.T) {
executor, sbc1, sbc2, _, ctx := createExecutorEnv(t)
result1 := []*sqltypes.Result{{
Expand Down
73 changes: 73 additions & 0 deletions go/vt/vtgate/planbuilder/testdata/filter_cases.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -797,6 +797,79 @@
]
}
},
{
"comment": "Disjunction of conjunctions with 4 or more disjunctions",
"query": "select id from user where (col = 'aa' AND name = 'bb') OR (col = 'cc' AND name = 'dd') OR (col = 'ee' AND name = 'ff') OR (col = 'gg' AND name = 'hh')",
"plan": {
"QueryType": "SELECT",
"Original": "select id from user where (col = 'aa' AND name = 'bb') OR (col = 'cc' AND name = 'dd') OR (col = 'ee' AND name = 'ff') OR (col = 'gg' AND name = 'hh')",
"Instructions": {
"OperatorType": "Route",
"Variant": "Scatter",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"FieldQuery": "select id from `user` where 1 != 1",
"Query": "select id from `user` where col = 'aa' and `name` = 'bb' or col = 'cc' and `name` = 'dd' or col = 'ee' and `name` = 'ff' or col = 'gg' and `name` = 'hh'",
"Table": "`user`"
},
"TablesUsed": [
"user.user"
]
}
},
{
"comment": "Disjunction of conjunctions with 3 or less disjunctions",
"query": "select id from user where (col = 'aa' AND name = 'bb') OR (col = 'cc' AND name = 'dd') OR (col = 'ee' AND name = 'ff')",
"plan": {
"QueryType": "SELECT",
"Original": "select id from user where (col = 'aa' AND name = 'bb') OR (col = 'cc' AND name = 'dd') OR (col = 'ee' AND name = 'ff')",
"Instructions": {
"OperatorType": "VindexLookup",
"Variant": "IN",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"Values": [
"(VARCHAR(\"bb\"), VARCHAR(\"dd\"), VARCHAR(\"ff\"))"
],
"Vindex": "name_user_map",
"Inputs": [
{
"OperatorType": "Route",
"Variant": "IN",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1",
"Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals",
"Table": "name_user_vdx",
"Values": [
"::name"
],
"Vindex": "user_index"
},
{
"OperatorType": "Route",
"Variant": "ByDestination",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"FieldQuery": "select id from `user` where 1 != 1",
"Query": "select id from `user` where col in ('aa', 'cc', 'ee') and (col in ('aa', 'cc') or `name` = 'ff') and (col = 'aa' or `name` = 'dd' or col = 'ee') and (col = 'aa' or `name` = 'dd' or `name` = 'ff') and (`name` = 'bb' or col = 'cc' or col = 'ee') and (`name` = 'bb' or col = 'cc' or `name` = 'ff') and (`name` in ('bb', 'dd') or col = 'ee') and `name` in ::__vals",
"Table": "`user`"
}
]
},
"TablesUsed": [
"user.user"
]
}
},
{
"comment": "Single table complex in clause",
"query": "select id from user where name in (col, 'bb')",
Expand Down

0 comments on commit c58abd9

Please sign in to comment.