From 694a0cf8d9e51aa4b130ad1656c4009427eb0e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Taylor?= Date: Thu, 4 Jul 2024 14:30:54 +0200 Subject: [PATCH] Prevent Early Ordering Pushdown to Enable Aggregation Pushdown to MySQL (#16278) Signed-off-by: Andres Taylor --- .../queries/benchmark/benchmark_test.go | 51 + .../queries/benchmark/sharded_schema.sql | 37 +- .../vtgate/queries/benchmark/vschema.json | 16 + .../planbuilder/operator_transformers.go | 7 +- .../planbuilder/operators/SQL_builder.go | 2 +- .../operators/aggregation_pushing.go | 2 +- .../planbuilder/operators/aggregator.go | 2 +- .../planbuilder/operators/apply_join.go | 75 +- go/vt/vtgate/planbuilder/operators/horizon.go | 2 +- .../planbuilder/operators/offset_planning.go | 81 +- .../operators/projection_pushing.go | 10 +- .../planbuilder/operators/query_planning.go | 27 +- .../planbuilder/operators/queryprojection.go | 54 +- .../vtgate/planbuilder/operators/rewriters.go | 1 + .../operators/subquery_planning.go | 4 +- .../plancontext/planning_context.go | 37 + go/vt/vtgate/planbuilder/rewrite.go | 3 +- .../planbuilder/testdata/aggr_cases.json | 97 +- .../testdata/postprocess_cases.json | 4 +- .../planbuilder/testdata/tpch_cases.json | 1158 ++++++++++------- 20 files changed, 1008 insertions(+), 662 deletions(-) diff --git a/go/test/endtoend/vtgate/queries/benchmark/benchmark_test.go b/go/test/endtoend/vtgate/queries/benchmark/benchmark_test.go index 9a064c1769a..b76ae5a35c7 100644 --- a/go/test/endtoend/vtgate/queries/benchmark/benchmark_test.go +++ b/go/test/endtoend/vtgate/queries/benchmark/benchmark_test.go @@ -32,6 +32,35 @@ type testQuery struct { intTyp []bool } +var deleteUser, deleteUserExtra = "delete from user", "delete from user_extra" + +func generateInserts(userSize int, userExtraSize int) (string, string) { + var userInserts []string + var userExtraInserts []string + + // Generate user table inserts + for i := 1; i <= userSize; i++ { + id := i + notShardingKey := i + typeValue := i % 5 // Just an example for type values + teamID := i%userExtraSize + 1 // To ensure team_id references user_extra id + userInserts = append(userInserts, fmt.Sprintf("(%d, %d, %d, %d)", id, notShardingKey, typeValue, teamID)) + } + + // Generate user_extra table inserts + for i := 1; i <= userExtraSize; i++ { + id := i + notShardingKey := i + colValue := fmt.Sprintf("col_value_%d", i) + userExtraInserts = append(userExtraInserts, fmt.Sprintf("(%d, %d, '%s')", id, notShardingKey, colValue)) + } + + userInsertStatement := fmt.Sprintf("INSERT INTO user (id, not_sharding_key, type, team_id) VALUES %s;", strings.Join(userInserts, ", ")) + userExtraInsertStatement := fmt.Sprintf("INSERT INTO user_extra (id, not_sharding_key, col) VALUES %s;", strings.Join(userExtraInserts, ", ")) + + return userInsertStatement, userExtraInsertStatement +} + func (tq *testQuery) getInsertQuery(rows int) string { var allRows []string for i := 0; i < rows; i++ { @@ -146,3 +175,25 @@ func BenchmarkShardedTblDeleteIn(b *testing.B) { }) } } + +func BenchmarkShardedAggrPushDown(b *testing.B) { + conn, closer := start(b) + defer closer() + + sizes := []int{100, 500, 1000} + + for _, user := range sizes { + for _, userExtra := range sizes { + insert1, insert2 := generateInserts(user, userExtra) + _ = utils.Exec(b, conn, insert1) + _ = utils.Exec(b, conn, insert2) + b.Run(fmt.Sprintf("user-%d-user_extra-%d", user, userExtra), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = utils.Exec(b, conn, "select sum(user.type) from user join user_extra on user.team_id = user_extra.id group by user_extra.id order by user_extra.id") + } + }) + _ = utils.Exec(b, conn, deleteUser) + _ = utils.Exec(b, conn, deleteUserExtra) + } + } +} diff --git a/go/test/endtoend/vtgate/queries/benchmark/sharded_schema.sql b/go/test/endtoend/vtgate/queries/benchmark/sharded_schema.sql index 850b6ffc15a..92f63c40f0f 100644 --- a/go/test/endtoend/vtgate/queries/benchmark/sharded_schema.sql +++ b/go/test/endtoend/vtgate/queries/benchmark/sharded_schema.sql @@ -1,16 +1,33 @@ create table tbl_no_lkp_vdx ( id bigint, - c1 varchar(50), - c2 varchar(50), - c3 varchar(50), - c4 varchar(50), - c5 varchar(50), - c6 varchar(50), - c7 varchar(50), - c8 varchar(50), - c9 varchar(50), + c1 varchar(50), + c2 varchar(50), + c3 varchar(50), + c4 varchar(50), + c5 varchar(50), + c6 varchar(50), + c7 varchar(50), + c8 varchar(50), + c9 varchar(50), c10 varchar(50), c11 varchar(50), c12 varchar(50) -) Engine = InnoDB; \ No newline at end of file +) Engine = InnoDB; + +create table user +( + id bigint, + not_sharding_key bigint, + type int, + team_id int, + primary key (id) +); + +create table user_extra +( + id bigint, + not_sharding_key bigint, + col varchar(50), + primary key (id) +); \ No newline at end of file diff --git a/go/test/endtoend/vtgate/queries/benchmark/vschema.json b/go/test/endtoend/vtgate/queries/benchmark/vschema.json index 4970e8b7437..efc854af8ff 100644 --- a/go/test/endtoend/vtgate/queries/benchmark/vschema.json +++ b/go/test/endtoend/vtgate/queries/benchmark/vschema.json @@ -13,6 +13,22 @@ "name": "xxhash" } ] + }, + "user": { + "column_vindexes": [ + { + "column": "id", + "name": "xxhash" + } + ] + }, + "user_extra": { + "column_vindexes": [ + { + "column": "id", + "name": "xxhash" + } + ] } } } \ No newline at end of file diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index da3f7348afd..546a9854f26 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -304,9 +304,14 @@ func transformAggregator(ctx *plancontext.PlanningContext, op *operators.Aggrega var groupByKeys []*engine.GroupByParams for _, aggr := range op.Aggregations { - if aggr.OpCode == opcode.AggregateUnassigned { + switch aggr.OpCode { + case opcode.AggregateUnassigned: return nil, vterrors.VT12001(fmt.Sprintf("in scatter query: aggregation function '%s'", sqlparser.String(aggr.Original))) + case opcode.AggregateUDF: + message := fmt.Sprintf("Aggregate UDF '%s' must be pushed down to MySQL", sqlparser.String(aggr.Original.Expr)) + return nil, vterrors.VT12001(message) } + aggrParam := engine.NewAggregateParam(aggr.OpCode, aggr.ColOffset, aggr.Alias, ctx.VSchema.Environment().CollationEnv()) aggrParam.Func = aggr.Func if gcFunc, isGc := aggrParam.Func.(*sqlparser.GroupConcatExpr); isGc && gcFunc.Separator == "" { diff --git a/go/vt/vtgate/planbuilder/operators/SQL_builder.go b/go/vt/vtgate/planbuilder/operators/SQL_builder.go index 0a2e545ea48..fef4ae2aee4 100644 --- a/go/vt/vtgate/planbuilder/operators/SQL_builder.go +++ b/go/vt/vtgate/planbuilder/operators/SQL_builder.go @@ -96,7 +96,7 @@ func (qb *queryBuilder) addPredicate(expr sqlparser.Expr) { switch stmt := qb.stmt.(type) { case *sqlparser.Select: - if ContainsAggr(qb.ctx, expr) { + if qb.ctx.ContainsAggr(expr) { addPred = stmt.AddHaving } else { addPred = stmt.AddWhere diff --git a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go index ca896587a29..671f4b78954 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go @@ -370,7 +370,6 @@ func pushAggregationThroughApplyJoin(ctx *plancontext.PlanningContext, rootAggr columns := &applyJoinColumns{} output, err := splitAggrColumnsToLeftAndRight(ctx, rootAggr, join, !join.JoinType.IsInner(), columns, lhs, rhs) - join.JoinColumns = columns if err != nil { // if we get this error, we just abort the splitting and fall back on simpler ways of solving the same query if errors.Is(err, errAbortAggrPushing) { @@ -378,6 +377,7 @@ func pushAggregationThroughApplyJoin(ctx *plancontext.PlanningContext, rootAggr } panic(err) } + join.JoinColumns = columns splitGroupingToLeftAndRight(ctx, rootAggr, lhs, rhs, join.JoinColumns) diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index 4a619b2e831..9db119bcaad 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -101,7 +101,7 @@ func (a *Aggregator) addColumnWithoutPushing(ctx *plancontext.PlanningContext, e case sqlparser.AggrFunc: aggr = createAggrFromAggrFunc(e, expr) case *sqlparser.FuncExpr: - if IsAggr(ctx, e) { + if ctx.IsAggr(e) { aggr = NewAggr(opcode.AggregateUDF, nil, expr, expr.As.String()) } else { aggr = NewAggr(opcode.AggregateAnyValue, nil, expr, expr.As.String()) diff --git a/go/vt/vtgate/planbuilder/operators/apply_join.go b/go/vt/vtgate/planbuilder/operators/apply_join.go index 0d214c545d1..cd8f7b94dd5 100644 --- a/go/vt/vtgate/planbuilder/operators/apply_join.go +++ b/go/vt/vtgate/planbuilder/operators/apply_join.go @@ -202,19 +202,11 @@ func (aj *ApplyJoin) GetOrdering(ctx *plancontext.PlanningContext) []OrderBy { return aj.LHS.GetOrdering(ctx) } -func joinColumnToExpr(column applyJoinColumn) sqlparser.Expr { - return column.Original -} - func (aj *ApplyJoin) getJoinColumnFor(ctx *plancontext.PlanningContext, orig *sqlparser.AliasedExpr, e sqlparser.Expr, addToGroupBy bool) (col applyJoinColumn) { - defer func() { - col.Original = orig.Expr - }() lhs := TableID(aj.LHS) rhs := TableID(aj.RHS) both := lhs.Merge(rhs) deps := ctx.SemTable.RecursiveDeps(e) - col.GroupBy = addToGroupBy switch { case deps.IsSolvedBy(lhs): @@ -224,9 +216,12 @@ func (aj *ApplyJoin) getJoinColumnFor(ctx *plancontext.PlanningContext, orig *sq case deps.IsSolvedBy(both): col = breakExpressionInLHSandRHS(ctx, e, TableID(aj.LHS)) default: - panic(vterrors.VT13002(sqlparser.String(e))) + panic(vterrors.VT13001(fmt.Sprintf("expression depends on tables outside this join: %s", sqlparser.String(e)))) } + col.GroupBy = addToGroupBy + col.Original = orig.Expr + return } @@ -289,7 +284,7 @@ func (aj *ApplyJoin) AddWSColumn(ctx *plancontext.PlanningContext, offset int, u if out >= 0 { aj.addOffset(out) } else { - col := aj.getJoinColumnFor(ctx, aeWrap(wsExpr), wsExpr, !ContainsAggr(ctx, wsExpr)) + col := aj.getJoinColumnFor(ctx, aeWrap(wsExpr), wsExpr, !ctx.ContainsAggr(wsExpr)) aj.JoinColumns.add(col) aj.planOffsetFor(ctx, col) } @@ -323,33 +318,15 @@ func (aj *ApplyJoin) planOffsets(ctx *plancontext.PlanningContext) Operator { } func (aj *ApplyJoin) planOffsetFor(ctx *plancontext.PlanningContext, col applyJoinColumn) { - if col.DTColName != nil { - // If DTColName is set, then we already pushed the parts of the expression down while planning. - // We need to use this name and ask the correct side of the join for it. Nothing else is required. - if col.IsPureLeft() { - offset := aj.LHS.AddColumn(ctx, true, col.GroupBy, aeWrap(col.DTColName)) - aj.addOffset(ToLeftOffset(offset)) - } else { - for _, lhsExpr := range col.LHSExprs { - offset := aj.LHS.AddColumn(ctx, true, col.GroupBy, aeWrap(lhsExpr.Expr)) - aj.Vars[lhsExpr.Name] = offset - } - offset := aj.RHS.AddColumn(ctx, true, col.GroupBy, aeWrap(col.DTColName)) - aj.addOffset(ToRightOffset(offset)) - } - return - } - for _, lhsExpr := range col.LHSExprs { - offset := aj.LHS.AddColumn(ctx, true, col.GroupBy, aeWrap(lhsExpr.Expr)) - if col.RHSExpr == nil { - // if we don't have an RHS expr, it means that this is a pure LHS expression - aj.addOffset(ToLeftOffset(offset)) - } else { + if col.IsPureLeft() { + offset := aj.LHS.AddColumn(ctx, true, col.GroupBy, aeWrap(col.GetPureLeftExpr())) + aj.addOffset(ToLeftOffset(offset)) + } else { + for _, lhsExpr := range col.LHSExprs { + offset := aj.LHS.AddColumn(ctx, true, col.GroupBy, aeWrap(lhsExpr.Expr)) aj.Vars[lhsExpr.Name] = offset } - } - if col.RHSExpr != nil { - offset := aj.RHS.AddColumn(ctx, true, col.GroupBy, aeWrap(col.RHSExpr)) + offset := aj.RHS.AddColumn(ctx, true, col.GroupBy, aeWrap(col.GetRHSExpr())) aj.addOffset(ToRightOffset(offset)) } } @@ -443,17 +420,17 @@ func (aj *ApplyJoin) findOrAddColNameBindVarName(ctx *plancontext.PlanningContex return bvName } -func (a *ApplyJoin) LHSColumnsNeeded(ctx *plancontext.PlanningContext) (needed sqlparser.Exprs) { +func (aj *ApplyJoin) LHSColumnsNeeded(ctx *plancontext.PlanningContext) (needed sqlparser.Exprs) { f := func(from BindVarExpr) sqlparser.Expr { return from.Expr } - for _, jc := range a.JoinColumns.columns { + for _, jc := range aj.JoinColumns.columns { needed = append(needed, slice.Map(jc.LHSExprs, f)...) } - for _, jc := range a.JoinPredicates.columns { + for _, jc := range aj.JoinPredicates.columns { needed = append(needed, slice.Map(jc.LHSExprs, f)...) } - needed = append(needed, slice.Map(a.ExtraLHSVars, f)...) + needed = append(needed, slice.Map(aj.ExtraLHSVars, f)...) return ctx.SemTable.Uniquify(needed) } @@ -462,7 +439,11 @@ func (jc applyJoinColumn) String() string { lhs := slice.Map(jc.LHSExprs, func(e BindVarExpr) string { return sqlparser.String(e.Expr) }) - return fmt.Sprintf("[%s | %s | %s]", strings.Join(lhs, ", "), rhs, sqlparser.String(jc.Original)) + if jc.DTColName == nil { + return fmt.Sprintf("[%s | %s | %s]", strings.Join(lhs, ", "), rhs, sqlparser.String(jc.Original)) + } + + return fmt.Sprintf("[%s | %s | %s | %s]", strings.Join(lhs, ", "), rhs, sqlparser.String(jc.Original), sqlparser.String(jc.DTColName)) } func (jc applyJoinColumn) IsPureLeft() bool { @@ -477,6 +458,20 @@ func (jc applyJoinColumn) IsMixedLeftAndRight() bool { return len(jc.LHSExprs) > 0 && jc.RHSExpr != nil } +func (jc applyJoinColumn) GetPureLeftExpr() sqlparser.Expr { + if jc.DTColName != nil { + return jc.DTColName + } + return jc.LHSExprs[0].Expr +} + +func (jc applyJoinColumn) GetRHSExpr() sqlparser.Expr { + if jc.DTColName != nil { + return jc.DTColName + } + return jc.RHSExpr +} + func (bve BindVarExpr) String() string { if bve.Name == "" { return sqlparser.String(bve.Expr) diff --git a/go/vt/vtgate/planbuilder/operators/horizon.go b/go/vt/vtgate/planbuilder/operators/horizon.go index c28bc2dd5f0..7539704c2a9 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon.go +++ b/go/vt/vtgate/planbuilder/operators/horizon.go @@ -100,7 +100,7 @@ func (h *Horizon) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser. } newExpr := ctx.RewriteDerivedTableExpression(expr, tableInfo) - if ContainsAggr(ctx, newExpr) { + if ctx.ContainsAggr(newExpr) { return newFilter(h, expr) } h.Source = h.Source.AddPredicate(ctx, newExpr) diff --git a/go/vt/vtgate/planbuilder/operators/offset_planning.go b/go/vt/vtgate/planbuilder/operators/offset_planning.go index eb92cdf0920..7a9cedccb89 100644 --- a/go/vt/vtgate/planbuilder/operators/offset_planning.go +++ b/go/vt/vtgate/planbuilder/operators/offset_planning.go @@ -21,7 +21,6 @@ import ( "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/engine/opcode" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" "vitess.io/vitess/go/vt/vtgate/semantics" ) @@ -96,62 +95,68 @@ func useOffsets(ctx *plancontext.PlanningContext, expr sqlparser.Expr, op Operat return rewritten.(sqlparser.Expr) } +func findAggregatorInSource(op Operator) *Aggregator { + // we'll just loop through the inputs until we find the aggregator + for { + aggr, ok := op.(*Aggregator) + if ok { + return aggr + } + inputs := op.Inputs() + if len(inputs) != 1 { + panic(vterrors.VT13001("unexpected multiple inputs")) + } + src := inputs[0] + _, isRoute := src.(*Route) + if isRoute { + panic(vterrors.VT13001("failed to find the aggregator")) + } + op = src + } +} + +// addColumnsToInput adds columns needed by an operator to its input. +// This happens only when the filter expression can be retrieved as an offset from the underlying mysql. func addColumnsToInput(ctx *plancontext.PlanningContext, root Operator) Operator { - // addColumnsToInput adds columns needed by an operator to its input. - // This happens only when the filter expression can be retrieved as an offset from the underlying mysql. + addColumnsNeededByFilter := func(in Operator, _ semantics.TableSet, _ bool) (Operator, *ApplyResult) { + addedCols := false filter, ok := in.(*Filter) if !ok { return in, NoRewrite } - proj, areOnTopOfProj := filter.Source.(selectExpressions) - if !areOnTopOfProj { - // not much we can do here - return in, NoRewrite - } - addedColumns := false - found := func(expr sqlparser.Expr, i int) {} - notFound := func(e sqlparser.Expr) { - _, addToGroupBy := e.(*sqlparser.ColName) - proj.addColumnWithoutPushing(ctx, aeWrap(e), addToGroupBy) - addedColumns = true + var neededAggrs []sqlparser.Expr + extractAggrs := func(cursor *sqlparser.CopyOnWriteCursor) { + node := cursor.Node() + if ctx.IsAggr(node) { + neededAggrs = append(neededAggrs, node.(sqlparser.Expr)) + } } - visitor := getOffsetRewritingVisitor(ctx, proj.FindCol, found, notFound) for _, expr := range filter.Predicates { - _ = sqlparser.CopyOnRewrite(expr, visitor, nil, ctx.SemTable.CopySemanticInfo) - } - if addedColumns { - return in, Rewrote("added columns because filter needs it") + _ = sqlparser.CopyOnRewrite(expr, dontEnterSubqueries, extractAggrs, nil) } - return in, NoRewrite - } - - // while we are out here walking the operator tree, if we find a UDF in an aggregation, we should fail - failUDFAggregation := func(in Operator, _ semantics.TableSet, _ bool) (Operator, *ApplyResult) { - aggrOp, ok := in.(*Aggregator) - if !ok { + if neededAggrs == nil { return in, NoRewrite } - for _, aggr := range aggrOp.Aggregations { - if aggr.OpCode == opcode.AggregateUDF { - // we don't support UDFs in aggregation if it's still above a route - message := fmt.Sprintf("Aggregate UDF '%s' must be pushed down to MySQL", sqlparser.String(aggr.Original.Expr)) - panic(vterrors.VT12001(message)) + + aggregator := findAggregatorInSource(filter.Source) + for _, aggr := range neededAggrs { + if aggregator.FindCol(ctx, aggr, false) == -1 { + aggregator.addColumnWithoutPushing(ctx, aeWrap(aggr), false) + addedCols = true } } - return in, NoRewrite - } - visitor := func(in Operator, _ semantics.TableSet, isRoot bool) (Operator, *ApplyResult) { - out, res := addColumnsNeededByFilter(in, semantics.EmptyTableSet(), isRoot) - failUDFAggregation(in, semantics.EmptyTableSet(), isRoot) - return out, res + if addedCols { + return in, Rewrote("added columns because filter needs it") + } + return in, NoRewrite } - return TopDown(root, TableID, visitor, stopAtRoute) + return TopDown(root, TableID, addColumnsNeededByFilter, stopAtRoute) } // isolateDistinctFromUnion will pull out the distinct from a union operator diff --git a/go/vt/vtgate/planbuilder/operators/projection_pushing.go b/go/vt/vtgate/planbuilder/operators/projection_pushing.go index d1baf6def62..b5c6a10bb78 100644 --- a/go/vt/vtgate/planbuilder/operators/projection_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/projection_pushing.go @@ -284,7 +284,11 @@ func pushProjectionInApplyJoin( src.LHS = createProjectionWithTheseColumns(ctx, src.LHS, lhs, p.DT) src.RHS = createProjectionWithTheseColumns(ctx, src.RHS, rhs, p.DT) - return src, Rewrote("split projection to either side of join") + message := "split projection to either side of join" + if p.DT != nil { + message += " " + p.DT.Alias + } + return src, Rewrote(message) } // splitProjectionAcrossJoin creates JoinPredicates for all projections, @@ -396,14 +400,14 @@ func exposeColumnsThroughDerivedTable(ctx *plancontext.PlanningContext, p *Proje lhsIDs := TableID(src.LHS) rhsIDs := TableID(src.RHS) - rewriteColumnsForJoin(ctx, src.JoinPredicates.columns, lhsIDs, rhsIDs, lhs, rhs) + rewriteColumnsForJoin(ctx, src.JoinPredicates.columns, lhsIDs, rhsIDs, lhs) } func rewriteColumnsForJoin( ctx *plancontext.PlanningContext, columns []applyJoinColumn, lhsIDs, rhsIDs semantics.TableSet, - lhs, rhs *projector, + lhs *projector, ) { for colIdx, column := range columns { for lhsIdx, bve := range column.LHSExprs { diff --git a/go/vt/vtgate/planbuilder/operators/query_planning.go b/go/vt/vtgate/planbuilder/operators/query_planning.go index 2602e39f87e..e88fb53edb3 100644 --- a/go/vt/vtgate/planbuilder/operators/query_planning.go +++ b/go/vt/vtgate/planbuilder/operators/query_planning.go @@ -592,14 +592,23 @@ ordering: return true } +// pushOrderingUnderAggr pushes the ORDER BY clause under the aggregation if possible, +// to optimize the query plan by aligning the GROUP BY and ORDER BY clauses and +// potentially removing redundant ORDER BY clauses. func pushOrderingUnderAggr(ctx *plancontext.PlanningContext, order *Ordering, aggregator *Aggregator) (Operator, *ApplyResult) { - // If Aggregator is a derived table, then we should rewrite the ordering before pushing. + // Avoid pushing down too early to allow for aggregation pushdown to MySQL + if !reachedPhase(ctx, delegateAggregation) { + return order, NoRewrite + } + + // Rewrite ORDER BY if Aggregator is a derived table if aggregator.isDerived() { for idx, orderExpr := range order.Order { ti, err := ctx.SemTable.TableInfoFor(aggregator.DT.TableID) if err != nil { panic(err) } + // Rewrite expressions in ORDER BY to match derived table columns newOrderExpr := orderExpr.Map(func(expr sqlparser.Expr) sqlparser.Expr { return semantics.RewriteDerivedTableExpression(expr, ti) }) @@ -607,9 +616,7 @@ func pushOrderingUnderAggr(ctx *plancontext.PlanningContext, order *Ordering, ag } } - // Step 1: Align the GROUP BY and ORDER BY. - // Reorder the GROUP BY columns to match the ORDER BY columns. - // Since the GB clause is a set, we can reorder these columns freely. + // Align GROUP BY with ORDER BY by reordering GROUP BY columns to match ORDER BY var newGrouping []GroupBy used := make([]bool, len(aggregator.Grouping)) for _, orderExpr := range order.Order { @@ -621,11 +628,8 @@ func pushOrderingUnderAggr(ctx *plancontext.PlanningContext, order *Ordering, ag } } - // Step 2: Add any missing columns from the ORDER BY. - // The ORDER BY column is not a set, but we can add more elements - // to the end without changing the semantics of the query. + // Add any missing GROUP BY columns to ORDER BY if len(newGrouping) != len(aggregator.Grouping) { - // we are missing some groupings. We need to add them both to the new groupings list, but also to the ORDER BY for i, added := range used { if !added { groupBy := aggregator.Grouping[i] @@ -638,7 +642,7 @@ func pushOrderingUnderAggr(ctx *plancontext.PlanningContext, order *Ordering, ag aggregator.Grouping = newGrouping aggrSource, isOrdering := aggregator.Source.(*Ordering) if isOrdering { - // Transform the query plan tree: + // Optimize query plan by removing redundant ORDER BY // From: Ordering(1) To: Aggregation // | | // Aggregation Ordering(1) @@ -646,11 +650,8 @@ func pushOrderingUnderAggr(ctx *plancontext.PlanningContext, order *Ordering, ag // Ordering(2) // | // - // - // Remove Ordering(2) from the plan tree, as it's redundant - // after pushing down the higher ordering. order.Source = aggrSource.Source - aggrSource.Source = nil // removing from plan tree + aggrSource.Source = nil aggregator.Source = order return aggregator, Rewrote("push ordering under aggregation, removing extra ordering") } diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index 2290927aef1..548fc5aaa0b 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -19,7 +19,6 @@ package operators import ( "encoding/json" "fmt" - "io" "slices" "sort" "strings" @@ -169,7 +168,7 @@ func createQPFromSelect(ctx *plancontext.PlanningContext, sel *sqlparser.Select) qp.addGroupBy(ctx, sel.GroupBy) qp.addOrderBy(ctx, sel.OrderBy) if !qp.HasAggr && sel.Having != nil { - qp.HasAggr = ContainsAggr(ctx, sel.Having.Expr) + qp.HasAggr = ctx.ContainsAggr(sel.Having.Expr) } qp.calculateDistinct(ctx) @@ -183,7 +182,7 @@ func (qp *QueryProjection) addSelectExpressions(ctx *plancontext.PlanningContext col := SelectExpr{ Col: selExp, } - if ContainsAggr(ctx, selExp.Expr) { + if ctx.ContainsAggr(selExp.Expr) { col.Aggr = true qp.HasAggr = true } @@ -200,41 +199,6 @@ func (qp *QueryProjection) addSelectExpressions(ctx *plancontext.PlanningContext } } -func IsAggr(ctx *plancontext.PlanningContext, e sqlparser.SQLNode) bool { - switch node := e.(type) { - case sqlparser.AggrFunc: - return true - case *sqlparser.FuncExpr: - return node.Name.EqualsAnyString(ctx.VSchema.GetAggregateUDFs()) - } - - return false -} - -func ContainsAggr(ctx *plancontext.PlanningContext, e sqlparser.SQLNode) (hasAggr bool) { - _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { - switch node.(type) { - case *sqlparser.Offset: - // offsets here indicate that a possible aggregation has already been handled by an input, - // so we don't need to worry about aggregation in the original - return false, nil - case sqlparser.AggrFunc: - hasAggr = true - return false, io.EOF - case *sqlparser.Subquery: - return false, nil - case *sqlparser.FuncExpr: - if IsAggr(ctx, node) { - hasAggr = true - return false, io.EOF - } - } - - return true, nil - }, e) - return -} - // createQPFromUnion creates the QueryProjection for the input *sqlparser.Union func createQPFromUnion(ctx *plancontext.PlanningContext, union *sqlparser.Union) *QueryProjection { qp := &QueryProjection{} @@ -277,7 +241,7 @@ func (qp *QueryProjection) addOrderBy(ctx *plancontext.PlanningContext, orderBy Inner: ctx.SemTable.Clone(order).(*sqlparser.Order), SimplifiedExpr: order.Expr, }) - canPushSorting = canPushSorting && !ContainsAggr(ctx, order.Expr) + canPushSorting = canPushSorting && !ctx.ContainsAggr(order.Expr) } } @@ -446,7 +410,7 @@ func (qp *QueryProjection) AggregationExpressions(ctx *plancontext.PlanningConte panic(err) } - if !ContainsAggr(ctx, expr.Col) { + if !ctx.ContainsAggr(expr.Col) { getExpr, err := expr.GetExpr() if err != nil { panic(err) @@ -457,7 +421,7 @@ func (qp *QueryProjection) AggregationExpressions(ctx *plancontext.PlanningConte } continue } - if !IsAggr(ctx, aliasedExpr.Expr) && !allowComplexExpression { + if !ctx.IsAggr(aliasedExpr.Expr) && !allowComplexExpression { panic(vterrors.VT12001("in scatter query: complex aggregate expression")) } @@ -486,14 +450,14 @@ func (qp *QueryProjection) extractAggr( addAggr(aggrFunc) return false } - if IsAggr(ctx, node) { + if ctx.IsAggr(node) { // If we are here, we have a function that is an aggregation but not parsed into an AggrFunc. // This is the case for UDFs - we have to be careful with these because we can't evaluate them in VTGate. aggr := NewAggr(opcode.AggregateUDF, nil, aeWrap(ex), "") addAggr(aggr) return false } - if ContainsAggr(ctx, node) { + if ctx.ContainsAggr(node) { makeComplex() return true } @@ -521,7 +485,7 @@ orderBy: } qp.SelectExprs = append(qp.SelectExprs, SelectExpr{ Col: &sqlparser.AliasedExpr{Expr: orderExpr}, - Aggr: ContainsAggr(ctx, orderExpr), + Aggr: ctx.ContainsAggr(orderExpr), }) qp.AddedColumn++ } @@ -702,7 +666,7 @@ func (qp *QueryProjection) useGroupingOverDistinct(ctx *plancontext.PlanningCont func checkForInvalidGroupingExpressions(ctx *plancontext.PlanningContext, expr sqlparser.Expr) { _ = sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) { - if IsAggr(ctx, node) { + if ctx.IsAggr(node) { panic(vterrors.VT03005(sqlparser.String(expr))) } _, isSubQ := node.(*sqlparser.Subquery) diff --git a/go/vt/vtgate/planbuilder/operators/rewriters.go b/go/vt/vtgate/planbuilder/operators/rewriters.go index d4d2c15f4ca..ab9fe66e368 100644 --- a/go/vt/vtgate/planbuilder/operators/rewriters.go +++ b/go/vt/vtgate/planbuilder/operators/rewriters.go @@ -232,6 +232,7 @@ func bottomUp( newOp, treeIdentity := rewriter(root, rootID, isRoot) anythingChanged = anythingChanged.Merge(treeIdentity) + return newOp, anythingChanged } diff --git a/go/vt/vtgate/planbuilder/operators/subquery_planning.go b/go/vt/vtgate/planbuilder/operators/subquery_planning.go index 5a0aed3f10d..aef54bd8c41 100644 --- a/go/vt/vtgate/planbuilder/operators/subquery_planning.go +++ b/go/vt/vtgate/planbuilder/operators/subquery_planning.go @@ -56,11 +56,11 @@ func isMergeable(ctx *plancontext.PlanningContext, query sqlparser.SelectStateme // if we have grouping, we have already checked that it's safe, and don't need to check for aggregations // but if we don't have groupings, we need to check if there are aggregations that will mess with us - if ContainsAggr(ctx, node.SelectExprs) { + if ctx.ContainsAggr(node.SelectExprs) { return false } - if ContainsAggr(ctx, node.Having) { + if ctx.ContainsAggr(node.Having) { return false } diff --git a/go/vt/vtgate/planbuilder/plancontext/planning_context.go b/go/vt/vtgate/planbuilder/plancontext/planning_context.go index 2f33539f858..7db192ab7f8 100644 --- a/go/vt/vtgate/planbuilder/plancontext/planning_context.go +++ b/go/vt/vtgate/planbuilder/plancontext/planning_context.go @@ -17,6 +17,8 @@ limitations under the License. package plancontext import ( + "io" + "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/sqlparser" @@ -232,3 +234,38 @@ func (ctx *PlanningContext) SQLTypeForExpr(e sqlparser.Expr) sqltypes.Type { } return t.Type() } + +func (ctx *PlanningContext) IsAggr(e sqlparser.SQLNode) bool { + switch node := e.(type) { + case sqlparser.AggrFunc: + return true + case *sqlparser.FuncExpr: + return node.Name.EqualsAnyString(ctx.VSchema.GetAggregateUDFs()) + } + + return false +} + +func (ctx *PlanningContext) ContainsAggr(e sqlparser.SQLNode) (hasAggr bool) { + _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { + switch node.(type) { + case *sqlparser.Offset: + // offsets here indicate that a possible aggregation has already been handled by an input, + // so we don't need to worry about aggregation in the original + return false, nil + case sqlparser.AggrFunc: + hasAggr = true + return false, io.EOF + case *sqlparser.Subquery: + return false, nil + case *sqlparser.FuncExpr: + if ctx.IsAggr(node) { + hasAggr = true + return false, io.EOF + } + } + + return true, nil + }, e) + return +} diff --git a/go/vt/vtgate/planbuilder/rewrite.go b/go/vt/vtgate/planbuilder/rewrite.go index 30423229038..ef9bf099de6 100644 --- a/go/vt/vtgate/planbuilder/rewrite.go +++ b/go/vt/vtgate/planbuilder/rewrite.go @@ -18,7 +18,6 @@ package planbuilder import ( "vitess.io/vitess/go/vt/sqlparser" - "vitess.io/vitess/go/vt/vtgate/planbuilder/operators" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" ) @@ -88,7 +87,7 @@ func (r *rewriter) rewriteHavingClause(node *sqlparser.Select) { exprs := sqlparser.SplitAndExpression(nil, node.Having.Expr) node.Having = nil for _, expr := range exprs { - if operators.ContainsAggr(r.ctx, expr) { + if r.ctx.ContainsAggr(expr) { node.AddHaving(expr) } else { node.AddWhere(expr) diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index 1ec5be4d49e..d51e2868f6c 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -1032,9 +1032,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by d, b, a, c, weight_string(d), weight_string(b), weight_string(a), weight_string(c)", + "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by a, b, c, d, weight_string(d), weight_string(b), weight_string(a), weight_string(c)", "OrderBy": "(3|5) ASC, (1|6) ASC, (0|7) ASC, (2|8) ASC", - "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by d, b, a, c, weight_string(d), weight_string(b), weight_string(a), weight_string(c) order by `user`.d asc, `user`.b asc, `user`.a asc, `user`.c asc", + "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by a, b, c, d, weight_string(d), weight_string(b), weight_string(a), weight_string(c) order by `user`.d asc, `user`.b asc, `user`.a asc, `user`.c asc", "Table": "`user`" } ] @@ -1064,9 +1064,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by d, b, a, c, weight_string(d), weight_string(b), weight_string(a), weight_string(c)", + "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by c, b, a, d, weight_string(d), weight_string(b), weight_string(a), weight_string(c)", "OrderBy": "(3|5) ASC, (1|6) ASC, (0|7) ASC, (2|8) ASC", - "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by d, b, a, c, weight_string(d), weight_string(b), weight_string(a), weight_string(c) order by `user`.d asc, `user`.b asc, `user`.a asc, `user`.c asc", + "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by c, b, a, d, weight_string(d), weight_string(b), weight_string(a), weight_string(c) order by `user`.d asc, `user`.b asc, `user`.a asc, `user`.c asc", "Table": "`user`" } ] @@ -1096,9 +1096,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` where 1 != 1 group by a, c, b, weight_string(a), weight_string(c), weight_string(b)", + "FieldQuery": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` where 1 != 1 group by c, b, a, weight_string(a), weight_string(c), weight_string(b)", "OrderBy": "(0|4) DESC, (2|5) DESC, (1|6) ASC", - "Query": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` group by a, c, b, weight_string(a), weight_string(c), weight_string(b) order by a desc, c desc, `user`.b asc", + "Query": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` group by c, b, a, weight_string(a), weight_string(c), weight_string(b) order by a desc, c desc, `user`.b asc", "Table": "`user`" } ] @@ -4759,8 +4759,8 @@ { "OperatorType": "Projection", "Expressions": [ - ":3 as col", - ":2 as intcol", + ":2 as col", + ":3 as intcol", "count(*) * count(*) as count(*)" ], "Inputs": [ @@ -4777,9 +4777,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*), u.intcol, u.col from `user` as u where 1 != 1 group by u.intcol, u.col", - "OrderBy": "1 ASC, 2 ASC", - "Query": "select count(*), u.intcol, u.col from `user` as u group by u.intcol, u.col order by u.intcol asc, u.col asc", + "FieldQuery": "select count(*), u.col, u.intcol from `user` as u where 1 != 1 group by u.col, u.intcol", + "OrderBy": "2 ASC, 1 ASC", + "Query": "select count(*), u.col, u.intcol from `user` as u group by u.col, u.intcol order by u.intcol asc, u.col asc", "Table": "`user`" }, { @@ -5324,8 +5324,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*), :user_id + user_extra.id, weight_string(:user_id + user_extra.id) from user_extra where 1 != 1 group by :user_id + user_extra.id", - "Query": "select count(*), :user_id + user_extra.id, weight_string(:user_id + user_extra.id) from user_extra group by :user_id + user_extra.id", + "FieldQuery": "select count(*), :user_id + user_extra.id, weight_string(:user_id + user_extra.id) from user_extra where 1 != 1 group by :user_id + user_extra.id, weight_string(:user_id + user_extra.id)", + "Query": "select count(*), :user_id + user_extra.id, weight_string(:user_id + user_extra.id) from user_extra group by :user_id + user_extra.id, weight_string(:user_id + user_extra.id)", "Table": "user_extra" } ] @@ -7145,5 +7145,76 @@ "user.user" ] } + }, + { + "comment": "should be able to push down aggregation", + "query": "select sum(user.type) from user join user_extra on user.team_id = user_extra.id group by user_extra.id order by user_extra.id", + "plan": { + "QueryType": "SELECT", + "Original": "select sum(user.type) from user join user_extra on user.team_id = user_extra.id group by user_extra.id order by user_extra.id", + "Instructions": { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "sum(0) AS sum(`user`.type)", + "GroupBy": "(1|2)", + "ResultColumns": 1, + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + "sum(`user`.type) * count(*) as sum(`user`.type)", + ":2 as id", + ":3 as weight_string(user_extra.id)" + ], + "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(2|3) ASC", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2", + "JoinVars": { + "user_team_id": 1 + }, + "TableName": "`user`_user_extra", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select sum(`user`.type), `user`.team_id from `user` where 1 != 1 group by `user`.team_id", + "Query": "select sum(`user`.type), `user`.team_id from `user` group by `user`.team_id", + "Table": "`user`" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select count(*), user_extra.id, weight_string(user_extra.id) from user_extra where 1 != 1 group by user_extra.id, weight_string(user_extra.id)", + "Query": "select count(*), user_extra.id, weight_string(user_extra.id) from user_extra where user_extra.id = :user_team_id group by user_extra.id, weight_string(user_extra.id)", + "Table": "user_extra" + } + ] + } + ] + } + ] + } + ] + }, + "TablesUsed": [ + "user.user", + "user.user_extra" + ] + } } ] diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json index 010e22c2108..8e2fd1e31cf 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json @@ -2185,9 +2185,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select foo, col, weight_string(foo) from `user` where 1 != 1 group by col, foo, weight_string(foo)", + "FieldQuery": "select foo, col, weight_string(foo) from `user` where 1 != 1 group by foo, col, weight_string(foo)", "OrderBy": "1 ASC, (0|2) ASC", - "Query": "select foo, col, weight_string(foo) from `user` where id between :vtg1 and :vtg2 group by col, foo, weight_string(foo) order by `user`.col asc, foo asc", + "Query": "select foo, col, weight_string(foo) from `user` where id between :vtg1 and :vtg2 group by foo, col, weight_string(foo) order by `user`.col asc, foo asc", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json index 559674ea8fc..442f4d6b8b6 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json @@ -566,149 +566,208 @@ "ResultColumns": 4, "Inputs": [ { - "OperatorType": "Sort", - "Variant": "Memory", - "OrderBy": "(0|4) ASC, (1|5) ASC, (2|6) ASC", + "OperatorType": "Projection", + "Expressions": [ + ":2 as supp_nation", + ":3 as cust_nation", + ":4 as l_year", + "sum(volume) * count(*) as revenue", + ":5 as weight_string(supp_nation)", + ":6 as weight_string(cust_nation)", + ":7 as weight_string(l_year)" + ], "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,L:1,L:2,L:4,R:1,L:5", - "JoinVars": { - "n1_n_name": 0, - "o_custkey": 3 - }, - "TableName": "lineitem_orders_supplier_nation_customer_nation", + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(2|5) ASC, (3|6) ASC, (4|7) ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,L:0,L:1,L:2,R:1,L:4", + "JoinColumnIndexes": "L:0,R:0,L:1,R:1,L:2,L:4,R:2,L:5", "JoinVars": { - "l_suppkey": 3 + "n1_n_name": 1, + "o_custkey": 3 }, - "TableName": "lineitem_orders_supplier_nation", + "TableName": "lineitem_orders_supplier_nation_customer_nation", "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,R:0,L:2,L:4", - "JoinVars": { - "l_orderkey": 3 - }, - "TableName": "lineitem_orders", + "OperatorType": "Projection", + "Expressions": [ + "sum(volume) * count(*) as revenue", + ":2 as supp_nation", + ":3 as l_year", + ":4 as o_custkey", + ":5 as weight_string(supp_nation)", + ":6 as weight_string(l_year)" + ], "Inputs": [ { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select shipping.l_year, shipping.volume, shipping.l_suppkey, shipping.l_orderkey, weight_string(shipping.l_year) from (select extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, l_suppkey as l_suppkey, l_orderkey as l_orderkey from lineitem where 1 != 1) as shipping where 1 != 1", - "Query": "select shipping.l_year, shipping.volume, shipping.l_suppkey, shipping.l_orderkey, weight_string(shipping.l_year) from (select extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, l_suppkey as l_suppkey, l_orderkey as l_orderkey from lineitem where l_shipdate between date('1995-01-01') and date('1996-12-31')) as shipping", - "Table": "lineitem" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,R:1,L:1,L:2,R:2,L:4", + "JoinVars": { + "l_suppkey": 3 }, - "FieldQuery": "select shipping.o_custkey from (select o_custkey as o_custkey from orders where 1 != 1) as shipping where 1 != 1", - "Query": "select shipping.o_custkey from (select o_custkey as o_custkey from orders where o_orderkey = :l_orderkey) as shipping", - "Table": "orders", - "Values": [ - ":l_orderkey" - ], - "Vindex": "hash" + "TableName": "lineitem_orders_supplier_nation", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + "sum(volume) * count(*) as revenue", + ":2 as l_year", + ":3 as o_custkey", + ":4 as l_suppkey", + ":5 as weight_string(l_year)" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,L:1,R:1,L:2,L:4", + "JoinVars": { + "l_orderkey": 3 + }, + "TableName": "lineitem_orders", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select sum(volume) as revenue, l_year, shipping.l_suppkey, shipping.l_orderkey, weight_string(l_year) from (select extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, l_suppkey as l_suppkey, l_orderkey as l_orderkey from lineitem where 1 != 1) as shipping where 1 != 1 group by l_year, shipping.l_suppkey, shipping.l_orderkey, weight_string(l_year)", + "Query": "select sum(volume) as revenue, l_year, shipping.l_suppkey, shipping.l_orderkey, weight_string(l_year) from (select extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, l_suppkey as l_suppkey, l_orderkey as l_orderkey from lineitem where l_shipdate between date('1995-01-01') and date('1996-12-31')) as shipping group by l_year, shipping.l_suppkey, shipping.l_orderkey, weight_string(l_year)", + "Table": "lineitem" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), shipping.o_custkey from (select o_custkey as o_custkey from orders where 1 != 1) as shipping where 1 != 1 group by shipping.o_custkey", + "Query": "select count(*), shipping.o_custkey from (select o_custkey as o_custkey from orders where o_orderkey = :l_orderkey) as shipping group by shipping.o_custkey", + "Table": "orders", + "Values": [ + ":l_orderkey" + ], + "Vindex": "hash" + } + ] + } + ] + }, + { + "OperatorType": "Projection", + "Expressions": [ + "count(*) * count(*) as count(*)", + ":2 as supp_nation", + ":3 as weight_string(supp_nation)" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2", + "JoinVars": { + "s_nationkey": 1 + }, + "TableName": "supplier_nation", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), shipping.s_nationkey from (select s_nationkey as s_nationkey from supplier where 1 != 1) as shipping where 1 != 1 group by shipping.s_nationkey", + "Query": "select count(*), shipping.s_nationkey from (select s_nationkey as s_nationkey from supplier where s_suppkey = :l_suppkey) as shipping group by shipping.s_nationkey", + "Table": "supplier", + "Values": [ + ":l_suppkey" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), supp_nation, weight_string(supp_nation) from (select n1.n_name as supp_nation from nation as n1 where 1 != 1) as shipping where 1 != 1 group by supp_nation, weight_string(supp_nation)", + "Query": "select count(*), supp_nation, weight_string(supp_nation) from (select n1.n_name as supp_nation from nation as n1 where n1.n_nationkey = :s_nationkey) as shipping group by supp_nation, weight_string(supp_nation)", + "Table": "nation", + "Values": [ + ":s_nationkey" + ], + "Vindex": "hash" + } + ] + } + ] + } + ] } ] }, { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0,R:1", - "JoinVars": { - "s_nationkey": 0 - }, - "TableName": "supplier_nation", + "OperatorType": "Projection", + "Expressions": [ + "count(*) * count(*) as count(*)", + ":2 as cust_nation", + ":3 as weight_string(cust_nation)" + ], "Inputs": [ { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select shipping.s_nationkey from (select s_nationkey as s_nationkey from supplier where 1 != 1) as shipping where 1 != 1", - "Query": "select shipping.s_nationkey from (select s_nationkey as s_nationkey from supplier where s_suppkey = :l_suppkey) as shipping", - "Table": "supplier", - "Values": [ - ":l_suppkey" - ], - "Vindex": "hash" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2", + "JoinVars": { + "c_nationkey": 1 }, - "FieldQuery": "select shipping.supp_nation, weight_string(shipping.supp_nation) from (select n1.n_name as supp_nation from nation as n1 where 1 != 1) as shipping where 1 != 1", - "Query": "select shipping.supp_nation, weight_string(shipping.supp_nation) from (select n1.n_name as supp_nation from nation as n1 where n1.n_nationkey = :s_nationkey) as shipping", - "Table": "nation", - "Values": [ - ":s_nationkey" - ], - "Vindex": "hash" + "TableName": "customer_nation", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), shipping.c_nationkey from (select c_nationkey as c_nationkey from customer where 1 != 1) as shipping where 1 != 1 group by shipping.c_nationkey", + "Query": "select count(*), shipping.c_nationkey from (select c_nationkey as c_nationkey from customer where c_custkey = :o_custkey) as shipping group by shipping.c_nationkey", + "Table": "customer", + "Values": [ + ":o_custkey" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), cust_nation, weight_string(cust_nation) from (select n2.n_name as cust_nation from nation as n2 where 1 != 1) as shipping where 1 != 1 group by cust_nation, weight_string(cust_nation)", + "Query": "select count(*), cust_nation, weight_string(cust_nation) from (select n2.n_name as cust_nation from nation as n2 where (:n1_n_name = 'FRANCE' and n2.n_name = 'GERMANY' or :n1_n_name = 'GERMANY' and n2.n_name = 'FRANCE') and n2.n_nationkey = :c_nationkey) as shipping group by cust_nation, weight_string(cust_nation)", + "Table": "nation", + "Values": [ + ":c_nationkey" + ], + "Vindex": "hash" + } + ] } ] } ] - }, - { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0,R:1", - "JoinVars": { - "c_nationkey": 0 - }, - "TableName": "customer_nation", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select shipping.c_nationkey from (select c_nationkey as c_nationkey from customer where 1 != 1) as shipping where 1 != 1", - "Query": "select shipping.c_nationkey from (select c_nationkey as c_nationkey from customer where c_custkey = :o_custkey) as shipping", - "Table": "customer", - "Values": [ - ":o_custkey" - ], - "Vindex": "hash" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select shipping.cust_nation, weight_string(shipping.cust_nation) from (select n2.n_name as cust_nation from nation as n2 where 1 != 1) as shipping where 1 != 1", - "Query": "select shipping.cust_nation, weight_string(shipping.cust_nation) from (select n2.n_name as cust_nation from nation as n2 where (:n1_n_name = 'FRANCE' and n2.n_name = 'GERMANY' or :n1_n_name = 'GERMANY' and n2.n_name = 'FRANCE') and n2.n_nationkey = :c_nationkey) as shipping", - "Table": "nation", - "Values": [ - ":c_nationkey" - ], - "Vindex": "hash" - } - ] } ] } @@ -746,200 +805,245 @@ "ResultColumns": 3, "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "0,3,1,4", + "OperatorType": "Projection", + "Expressions": [ + ":3 as o_year", + "sum(case when nation = 'BRAZIL' then volume else 0 end) * count(*) as sum(case when nation = 'BRAZIL' then volume else 0 end)", + "sum(volume) * count(*) as sum(volume)", + ":4 as weight_string(o_year)" + ], "Inputs": [ { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "(0|4) ASC", + "OrderBy": "(3|4) ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,L:0,L:1,L:3,R:1", + "JoinColumnIndexes": "L:0,R:0,L:1,R:1,R:2", "JoinVars": { "l_orderkey": 2 }, "TableName": "lineitem_part_supplier_nation_orders_customer_nation_region", "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,L:1,R:1", - "JoinVars": { - "l_suppkey": 2, - "volume": 0 - }, - "TableName": "lineitem_part_supplier_nation", + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "sum(0) AS sum(case when nation = 'BRAZIL' then volume else 0 end), sum(1) AS sum(volume)", + "GroupBy": "(2|3)", + "ResultColumns": 3, "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,L:2", + "JoinColumnIndexes": "R:1,L:0,L:1,L:3", "JoinVars": { - "l_partkey": 3 + "l_suppkey": 2, + "volume": 0 }, - "TableName": "lineitem_part", + "TableName": "lineitem_part_supplier_nation", "Inputs": [ { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "main", - "Sharded": true + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,L:1,L:2,L:4", + "JoinVars": { + "l_partkey": 3 }, - "FieldQuery": "select all_nations.volume, all_nations.l_orderkey, all_nations.l_suppkey, all_nations.l_partkey from (select l_extendedprice * (1 - l_discount) as volume, l_orderkey as l_orderkey, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where 1 != 1) as all_nations where 1 != 1", - "Query": "select all_nations.volume, all_nations.l_orderkey, all_nations.l_suppkey, all_nations.l_partkey from (select l_extendedprice * (1 - l_discount) as volume, l_orderkey as l_orderkey, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem) as all_nations", - "Table": "lineitem" + "TableName": "lineitem_part", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select all_nations.volume, all_nations.l_orderkey, all_nations.l_suppkey, all_nations.l_partkey, weight_string(all_nations.l_orderkey) from (select l_extendedprice * (1 - l_discount) as volume, l_orderkey as l_orderkey, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where 1 != 1) as all_nations where 1 != 1", + "OrderBy": "(1|4) ASC", + "Query": "select all_nations.volume, all_nations.l_orderkey, all_nations.l_suppkey, all_nations.l_partkey, weight_string(all_nations.l_orderkey) from (select l_extendedprice * (1 - l_discount) as volume, l_orderkey as l_orderkey, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem) as all_nations order by all_nations.l_orderkey asc", + "Table": "lineitem" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select 1 from part where 1 != 1", + "Query": "select 1 from part where p_type = 'ECONOMY ANODIZED STEEL' and p_partkey = :l_partkey", + "Table": "part", + "Values": [ + ":l_partkey" + ], + "Vindex": "hash" + } + ] }, { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,R:1", + "JoinVars": { + "s_nationkey": 0 }, - "FieldQuery": "select 1 from part where 1 != 1", - "Query": "select 1 from part where p_type = 'ECONOMY ANODIZED STEEL' and p_partkey = :l_partkey", - "Table": "part", - "Values": [ - ":l_partkey" - ], - "Vindex": "hash" - } - ] - }, - { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0,R:1", - "JoinVars": { - "s_nationkey": 0 - }, - "TableName": "supplier_nation", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select all_nations.s_nationkey from (select s_nationkey as s_nationkey from supplier where 1 != 1) as all_nations where 1 != 1", - "Query": "select all_nations.s_nationkey from (select s_nationkey as s_nationkey from supplier where s_suppkey = :l_suppkey) as all_nations", - "Table": "supplier", - "Values": [ - ":l_suppkey" - ], - "Vindex": "hash" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select all_nations.nation, case when nation = 'BRAZIL' then :volume else 0 end from (select n2.n_name as nation from nation as n2 where 1 != 1) as all_nations where 1 != 1", - "Query": "select all_nations.nation, case when nation = 'BRAZIL' then :volume else 0 end from (select n2.n_name as nation from nation as n2 where n2.n_nationkey = :s_nationkey) as all_nations", - "Table": "nation", - "Values": [ - ":s_nationkey" - ], - "Vindex": "hash" + "TableName": "supplier_nation", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select all_nations.s_nationkey from (select s_nationkey as s_nationkey from supplier where 1 != 1) as all_nations where 1 != 1", + "Query": "select all_nations.s_nationkey from (select s_nationkey as s_nationkey from supplier where s_suppkey = :l_suppkey) as all_nations", + "Table": "supplier", + "Values": [ + ":l_suppkey" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select all_nations.nation, case when nation = 'BRAZIL' then :volume else 0 end from (select n2.n_name as nation from nation as n2 where 1 != 1) as all_nations where 1 != 1", + "Query": "select all_nations.nation, case when nation = 'BRAZIL' then :volume else 0 end from (select n2.n_name as nation from nation as n2 where n2.n_nationkey = :s_nationkey) as all_nations", + "Table": "nation", + "Values": [ + ":s_nationkey" + ], + "Vindex": "hash" + } + ] } ] } ] }, { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "L:0,L:2", - "JoinVars": { - "c_nationkey": 1 - }, - "TableName": "orders_customer_nation_region", + "OperatorType": "Projection", + "Expressions": [ + "count(*) * count(*) as count(*)", + ":2 as o_year", + ":3 as weight_string(o_year)" + ], "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,L:2", - "JoinVars": { - "o_custkey": 1 - }, - "TableName": "orders_customer", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select all_nations.o_year, all_nations.o_custkey, weight_string(all_nations.o_year) from (select extract(year from o_orderdate) as o_year, o_custkey as o_custkey from orders where 1 != 1) as all_nations where 1 != 1", - "Query": "select all_nations.o_year, all_nations.o_custkey, weight_string(all_nations.o_year) from (select extract(year from o_orderdate) as o_year, o_custkey as o_custkey from orders where o_orderdate between date'1995-01-01' and date('1996-12-31') and o_orderkey = :l_orderkey) as all_nations", - "Table": "orders", - "Values": [ - ":l_orderkey" - ], - "Vindex": "hash" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select all_nations.c_nationkey from (select c_nationkey as c_nationkey from customer where 1 != 1) as all_nations where 1 != 1", - "Query": "select all_nations.c_nationkey from (select c_nationkey as c_nationkey from customer where c_custkey = :o_custkey) as all_nations", - "Table": "customer", - "Values": [ - ":o_custkey" - ], - "Vindex": "hash" - } - ] - }, - { - "OperatorType": "Join", - "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,L:1,L:3", "JoinVars": { - "n1_n_regionkey": 0 + "c_nationkey": 2 }, - "TableName": "nation_region", + "TableName": "orders_customer_nation_region", "Inputs": [ { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select n1.n_regionkey from nation as n1 where 1 != 1", - "Query": "select n1.n_regionkey from nation as n1 where n1.n_nationkey = :c_nationkey", - "Table": "nation", - "Values": [ - ":c_nationkey" + "OperatorType": "Projection", + "Expressions": [ + "count(*) * count(*) as count(*)", + ":2 as o_year", + ":3 as c_nationkey", + ":4 as weight_string(o_year)" ], - "Vindex": "hash" + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,L:1,R:1,L:3", + "JoinVars": { + "o_custkey": 2 + }, + "TableName": "orders_customer", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), o_year, all_nations.o_custkey, weight_string(o_year) from (select extract(year from o_orderdate) as o_year, o_custkey as o_custkey from orders where 1 != 1) as all_nations where 1 != 1 group by o_year, all_nations.o_custkey, weight_string(o_year)", + "Query": "select count(*), o_year, all_nations.o_custkey, weight_string(o_year) from (select extract(year from o_orderdate) as o_year, o_custkey as o_custkey from orders where o_orderdate between date'1995-01-01' and date('1996-12-31') and o_orderkey = :l_orderkey) as all_nations group by o_year, all_nations.o_custkey, weight_string(o_year)", + "Table": "orders", + "Values": [ + ":l_orderkey" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), all_nations.c_nationkey from (select c_nationkey as c_nationkey from customer where 1 != 1) as all_nations where 1 != 1 group by all_nations.c_nationkey", + "Query": "select count(*), all_nations.c_nationkey from (select c_nationkey as c_nationkey from customer where c_custkey = :o_custkey) as all_nations group by all_nations.c_nationkey", + "Table": "customer", + "Values": [ + ":o_custkey" + ], + "Vindex": "hash" + } + ] + } + ] }, { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select 1 from region where 1 != 1", - "Query": "select 1 from region where r_name = 'AMERICA' and r_regionkey = :n1_n_regionkey", - "Table": "region", - "Values": [ - ":n1_n_regionkey" + "OperatorType": "Projection", + "Expressions": [ + "count(*) * count(*) as count(*)" ], - "Vindex": "hash" + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0", + "JoinVars": { + "n1_n_regionkey": 1 + }, + "TableName": "nation_region", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), n1.n_regionkey from nation as n1 where 1 != 1 group by n1.n_regionkey", + "Query": "select count(*), n1.n_regionkey from nation as n1 where n1.n_nationkey = :c_nationkey group by n1.n_regionkey", + "Table": "nation", + "Values": [ + ":c_nationkey" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*) from region where 1 != 1 group by .0", + "Query": "select count(*) from region where r_name = 'AMERICA' and r_regionkey = :n1_n_regionkey group by .0", + "Table": "region", + "Values": [ + ":n1_n_regionkey" + ], + "Vindex": "hash" + } + ] + } + ] } ] } @@ -980,61 +1084,144 @@ "ResultColumns": 3, "Inputs": [ { - "OperatorType": "Sort", - "Variant": "Memory", - "OrderBy": "(0|3) ASC, (1|4) DESC", + "OperatorType": "Projection", + "Expressions": [ + ":2 as nation", + ":3 as o_year", + "sum(amount) * count(*) as sum_profit", + ":4 as weight_string(nation)", + ":5 as weight_string(o_year)" + ], "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0,L:0,L:1,R:1,L:3", - "JoinVars": { - "l_suppkey": 2 - }, - "TableName": "orders_lineitem_part_partsupp_supplier_nation", + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(2|4) ASC, (3|5) DESC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,L:4,L:6", + "JoinColumnIndexes": "L:0,R:0,R:1,L:1,R:2,L:3", "JoinVars": { - "l_discount": 2, - "l_extendedprice": 1, - "l_partkey": 5, - "l_quantity": 3, - "l_suppkey": 4 + "l_suppkey": 2 }, - "TableName": "orders_lineitem_part_partsupp", + "TableName": "orders_lineitem_part_partsupp_supplier_nation", "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,R:1,R:2,R:3,R:4,L:2", - "JoinVars": { - "o_orderkey": 1 - }, - "TableName": "orders_lineitem_part", + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "sum(0) AS sum_profit", + "GroupBy": "(1|3), (2|4)", + "ResultColumns": 4, "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders where 1 != 1) as profit where 1 != 1", - "Query": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders) as profit", - "Table": "orders" - }, { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,L:2,L:3,L:4", + "JoinColumnIndexes": "R:0,L:0,L:4,L:6,L:7", "JoinVars": { - "l_partkey": 4 + "l_discount": 2, + "l_extendedprice": 1, + "l_partkey": 5, + "l_quantity": 3, + "l_suppkey": 4 }, - "TableName": "lineitem_part", + "TableName": "orders_lineitem_part_partsupp", "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(0|6) ASC, (4|7) ASC", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2,R:3,R:4,L:2,R:5", + "JoinVars": { + "o_orderkey": 1 + }, + "TableName": "orders_lineitem_part", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders where 1 != 1) as profit where 1 != 1", + "Query": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders) as profit", + "Table": "orders" + }, + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,L:1,L:2,L:3,L:4,L:5", + "JoinVars": { + "l_partkey": 4 + }, + "TableName": "lineitem_part", + "Inputs": [ + { + "OperatorType": "VindexLookup", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "Values": [ + ":o_orderkey" + ], + "Vindex": "lineitem_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select l_orderkey, l_linenumber from lineitem_map where 1 != 1", + "Query": "select l_orderkey, l_linenumber from lineitem_map where l_orderkey in ::__vals", + "Table": "lineitem_map", + "Values": [ + "::l_orderkey" + ], + "Vindex": "md5" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice, l_discount, l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where 1 != 1) as profit where 1 != 1", + "Query": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice, l_discount, l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where l_orderkey = :o_orderkey) as profit", + "Table": "lineitem" + } + ] + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select 1 from part where 1 != 1", + "Query": "select 1 from part where p_name like '%green%' and p_partkey = :l_partkey", + "Table": "part", + "Values": [ + ":l_partkey" + ], + "Vindex": "hash" + } + ] + } + ] + } + ] + }, { "OperatorType": "VindexLookup", "Variant": "EqualUnique", @@ -1043,9 +1230,9 @@ "Sharded": true }, "Values": [ - ":o_orderkey" + ":l_partkey" ], - "Vindex": "lineitem_map", + "Vindex": "partsupp_map", "Inputs": [ { "OperatorType": "Route", @@ -1054,11 +1241,11 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select l_orderkey, l_linenumber from lineitem_map where 1 != 1", - "Query": "select l_orderkey, l_linenumber from lineitem_map where l_orderkey in ::__vals", - "Table": "lineitem_map", + "FieldQuery": "select ps_partkey, ps_suppkey from partsupp_map where 1 != 1", + "Query": "select ps_partkey, ps_suppkey from partsupp_map where ps_partkey in ::__vals", + "Table": "partsupp_map", "Values": [ - "::l_orderkey" + "::ps_partkey" ], "Vindex": "md5" }, @@ -1069,11 +1256,47 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey from (select l_extendedprice, l_discount, l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where 1 != 1) as profit where 1 != 1", - "Query": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey from (select l_extendedprice, l_discount, l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where l_orderkey = :o_orderkey) as profit", - "Table": "lineitem" + "FieldQuery": "select profit.amount from (select :l_extendedprice * (1 - :l_discount) - ps_supplycost * :l_quantity as amount from partsupp where 1 != 1) as profit where 1 != 1", + "Query": "select profit.amount from (select :l_extendedprice * (1 - :l_discount) - ps_supplycost * :l_quantity as amount from partsupp where ps_partkey = :l_partkey and ps_suppkey = :l_suppkey) as profit", + "Table": "partsupp" } ] + } + ] + } + ] + }, + { + "OperatorType": "Projection", + "Expressions": [ + "count(*) * count(*) as count(*)", + ":2 as nation", + ":3 as weight_string(nation)" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2", + "JoinVars": { + "s_nationkey": 1 + }, + "TableName": "supplier_nation", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), profit.s_nationkey from (select s_nationkey as s_nationkey from supplier where 1 != 1) as profit where 1 != 1 group by profit.s_nationkey", + "Query": "select count(*), profit.s_nationkey from (select s_nationkey as s_nationkey from supplier where s_suppkey = :l_suppkey) as profit group by profit.s_nationkey", + "Table": "supplier", + "Values": [ + ":l_suppkey" + ], + "Vindex": "hash" }, { "OperatorType": "Route", @@ -1082,98 +1305,17 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select 1 from part where 1 != 1", - "Query": "select 1 from part where p_name like '%green%' and p_partkey = :l_partkey", - "Table": "part", + "FieldQuery": "select count(*), nation, weight_string(nation) from (select n_name as nation from nation where 1 != 1) as profit where 1 != 1 group by nation, weight_string(nation)", + "Query": "select count(*), nation, weight_string(nation) from (select n_name as nation from nation where n_nationkey = :s_nationkey) as profit group by nation, weight_string(nation)", + "Table": "nation", "Values": [ - ":l_partkey" + ":s_nationkey" ], "Vindex": "hash" } ] } ] - }, - { - "OperatorType": "VindexLookup", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "Values": [ - ":l_partkey" - ], - "Vindex": "partsupp_map", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "IN", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select ps_partkey, ps_suppkey from partsupp_map where 1 != 1", - "Query": "select ps_partkey, ps_suppkey from partsupp_map where ps_partkey in ::__vals", - "Table": "partsupp_map", - "Values": [ - "::ps_partkey" - ], - "Vindex": "md5" - }, - { - "OperatorType": "Route", - "Variant": "ByDestination", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select profit.amount from (select :l_extendedprice * (1 - :l_discount) - ps_supplycost * :l_quantity as amount from partsupp where 1 != 1) as profit where 1 != 1", - "Query": "select profit.amount from (select :l_extendedprice * (1 - :l_discount) - ps_supplycost * :l_quantity as amount from partsupp where ps_partkey = :l_partkey and ps_suppkey = :l_suppkey) as profit", - "Table": "partsupp" - } - ] - } - ] - }, - { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0,R:1", - "JoinVars": { - "s_nationkey": 0 - }, - "TableName": "supplier_nation", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select profit.s_nationkey from (select s_nationkey as s_nationkey from supplier where 1 != 1) as profit where 1 != 1", - "Query": "select profit.s_nationkey from (select s_nationkey as s_nationkey from supplier where s_suppkey = :l_suppkey) as profit", - "Table": "supplier", - "Values": [ - ":l_suppkey" - ], - "Vindex": "hash" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select profit.nation, weight_string(profit.nation) from (select n_name as nation from nation where 1 != 1) as profit where 1 != 1", - "Query": "select profit.nation, weight_string(profit.nation) from (select n_name as nation from nation where n_nationkey = :s_nationkey) as profit", - "Table": "nation", - "Values": [ - ":s_nationkey" - ], - "Vindex": "hash" } ] } @@ -1645,67 +1787,78 @@ "ResultColumns": 3, "Inputs": [ { - "OperatorType": "Sort", - "Variant": "Memory", - "OrderBy": "(0|3) ASC", + "OperatorType": "Projection", + "Expressions": [ + ":3 as l_shipmode", + "sum(case when o_orderpriority = '1-URGENT' or o_orderpriority = '2-HIGH' then 1 else 0 end) * count(*) as high_line_count", + "sum(case when o_orderpriority != '1-URGENT' and o_orderpriority != '2-HIGH' then 1 else 0 end) * count(*) as low_line_count", + ":4 as weight_string(l_shipmode)" + ], "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0,L:0,L:1,R:1", - "JoinVars": { - "o_orderkey": 2 - }, - "TableName": "orders_lineitem", + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(3|4) ASC", "Inputs": [ { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select case when o_orderpriority = '1-URGENT' or o_orderpriority = '2-HIGH' then 1 else 0 end, case when o_orderpriority != '1-URGENT' and o_orderpriority != '2-HIGH' then 1 else 0 end, o_orderkey from orders where 1 != 1", - "Query": "select case when o_orderpriority = '1-URGENT' or o_orderpriority = '2-HIGH' then 1 else 0 end, case when o_orderpriority != '1-URGENT' and o_orderpriority != '2-HIGH' then 1 else 0 end, o_orderkey from orders", - "Table": "orders" - }, - { - "OperatorType": "VindexLookup", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,L:1,R:1,R:2", + "JoinVars": { + "o_orderkey": 2 }, - "Values": [ - ":o_orderkey" - ], - "Vindex": "lineitem_map", + "TableName": "orders_lineitem", "Inputs": [ { "OperatorType": "Route", - "Variant": "IN", + "Variant": "Scatter", "Keyspace": { "Name": "main", "Sharded": true }, - "FieldQuery": "select l_orderkey, l_linenumber from lineitem_map where 1 != 1", - "Query": "select l_orderkey, l_linenumber from lineitem_map where l_orderkey in ::__vals", - "Table": "lineitem_map", - "Values": [ - "::l_orderkey" - ], - "Vindex": "md5" + "FieldQuery": "select sum(case when o_orderpriority = '1-URGENT' or o_orderpriority = '2-HIGH' then 1 else 0 end) as high_line_count, sum(case when o_orderpriority != '1-URGENT' and o_orderpriority != '2-HIGH' then 1 else 0 end) as low_line_count, o_orderkey from orders where 1 != 1 group by o_orderkey", + "Query": "select sum(case when o_orderpriority = '1-URGENT' or o_orderpriority = '2-HIGH' then 1 else 0 end) as high_line_count, sum(case when o_orderpriority != '1-URGENT' and o_orderpriority != '2-HIGH' then 1 else 0 end) as low_line_count, o_orderkey from orders group by o_orderkey", + "Table": "orders" }, { - "OperatorType": "Route", - "Variant": "ByDestination", + "OperatorType": "VindexLookup", + "Variant": "EqualUnique", "Keyspace": { "Name": "main", "Sharded": true }, - "FieldQuery": "select l_shipmode, weight_string(l_shipmode) from lineitem where 1 != 1", - "Query": "select l_shipmode, weight_string(l_shipmode) from lineitem where l_shipmode in ('MAIL', 'SHIP') and l_commitdate < l_receiptdate and l_shipdate < l_commitdate and l_receiptdate >= date('1994-01-01') and l_receiptdate < date('1994-01-01') + interval '1' year and l_orderkey = :o_orderkey", - "Table": "lineitem" + "Values": [ + ":o_orderkey" + ], + "Vindex": "lineitem_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select l_orderkey, l_linenumber from lineitem_map where 1 != 1", + "Query": "select l_orderkey, l_linenumber from lineitem_map where l_orderkey in ::__vals", + "Table": "lineitem_map", + "Values": [ + "::l_orderkey" + ], + "Vindex": "md5" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), l_shipmode, weight_string(l_shipmode) from lineitem where 1 != 1 group by l_shipmode, weight_string(l_shipmode)", + "Query": "select count(*), l_shipmode, weight_string(l_shipmode) from lineitem where l_shipmode in ('MAIL', 'SHIP') and l_commitdate < l_receiptdate and l_shipdate < l_commitdate and l_receiptdate >= date('1994-01-01') and l_receiptdate < date('1994-01-01') + interval '1' year and l_orderkey = :o_orderkey group by l_shipmode, weight_string(l_shipmode)", + "Table": "lineitem" + } + ] } ] } @@ -2078,25 +2231,34 @@ }, { "InputName": "Outer", - "OperatorType": "Filter", - "Predicate": ":__sq_has_values and o_orderkey in ::__sq1", + "OperatorType": "Projection", + "Expressions": [ + ":2 as c_name", + ":3 as c_custkey", + ":4 as o_orderkey", + ":5 as o_orderdate", + ":6 as o_totalprice", + "sum(l_quantity) * count(*) as sum(l_quantity)", + ":7 as weight_string(o_totalprice)", + ":8 as weight_string(o_orderdate)", + ":9 as weight_string(c_name)", + ":10 as weight_string(c_custkey)", + ":11 as weight_string(o_orderkey)" + ], "Inputs": [ { - "OperatorType": "Aggregate", - "Variant": "Ordered", - "Aggregates": "sum(5) AS sum(l_quantity)", - "GroupBy": "(4|6), (3|7), (0|8), (1|9), (2|10)", - "ResultColumns": 11, + "OperatorType": "Filter", + "Predicate": ":__sq_has_values and o_orderkey in ::__sq1", "Inputs": [ { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "(4|6) DESC, (3|7) ASC, (0|8) ASC, (1|9) ASC, (2|10) ASC", + "OrderBy": "(6|7) DESC, (5|8) ASC, (2|9) ASC, (3|10) ASC, (4|11) ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,R:1,R:2,R:3,R:4,L:0,R:5,R:6,R:7,R:8,R:9", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2,R:3,R:4,R:5,R:6,R:7,R:8,R:9,R:10", "JoinVars": { "l_orderkey": 1 }, @@ -2109,48 +2271,66 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select l_quantity, l_orderkey from lineitem where 1 != 1", - "Query": "select l_quantity, l_orderkey from lineitem", + "FieldQuery": "select sum(l_quantity), l_orderkey from lineitem where 1 != 1 group by l_orderkey", + "Query": "select sum(l_quantity), l_orderkey from lineitem group by l_orderkey", "Table": "lineitem" }, { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0,R:1,L:0,L:1,L:2,L:3,L:4,R:2,R:3,L:5", - "JoinVars": { - "o_custkey": 6 - }, - "TableName": "orders_customer", + "OperatorType": "Projection", + "Expressions": [ + "count(*) * count(*) as count(*)", + ":2 as c_name", + ":3 as c_custkey", + ":4 as o_orderkey", + ":5 as o_orderdate", + ":6 as o_totalprice", + ":7 as weight_string(o_totalprice)", + ":8 as weight_string(o_orderdate)", + ":9 as weight_string(c_name)", + ":10 as weight_string(c_custkey)", + ":11 as weight_string(o_orderkey)" + ], "Inputs": [ { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select o_orderkey, o_orderdate, o_totalprice, weight_string(o_totalprice), weight_string(o_orderdate), weight_string(o_orderkey), o_custkey from orders where 1 != 1", - "Query": "select o_orderkey, o_orderdate, o_totalprice, weight_string(o_totalprice), weight_string(o_orderdate), weight_string(o_orderkey), o_custkey from orders where o_orderkey = :l_orderkey", - "Table": "orders", - "Values": [ - ":l_orderkey" - ], - "Vindex": "hash" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2,L:1,L:2,L:3,L:5,L:6,R:3,R:4,L:7", + "JoinVars": { + "o_custkey": 4 }, - "FieldQuery": "select c_name, c_custkey, weight_string(c_name), weight_string(c_custkey) from customer where 1 != 1", - "Query": "select c_name, c_custkey, weight_string(c_name), weight_string(c_custkey) from customer where c_custkey = :o_custkey", - "Table": "customer", - "Values": [ - ":o_custkey" - ], - "Vindex": "hash" + "TableName": "orders_customer", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), o_orderkey, o_orderdate, o_totalprice, o_custkey, weight_string(o_totalprice), weight_string(o_orderdate), weight_string(o_orderkey) from orders where 1 != 1 group by o_orderkey, o_orderdate, o_totalprice, o_custkey, weight_string(o_totalprice), weight_string(o_orderdate), weight_string(o_orderkey)", + "Query": "select count(*), o_orderkey, o_orderdate, o_totalprice, o_custkey, weight_string(o_totalprice), weight_string(o_orderdate), weight_string(o_orderkey) from orders where o_orderkey = :l_orderkey group by o_orderkey, o_orderdate, o_totalprice, o_custkey, weight_string(o_totalprice), weight_string(o_orderdate), weight_string(o_orderkey)", + "Table": "orders", + "Values": [ + ":l_orderkey" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), c_name, c_custkey, weight_string(c_name), weight_string(c_custkey) from customer where 1 != 1 group by c_name, c_custkey, weight_string(c_name), weight_string(c_custkey)", + "Query": "select count(*), c_name, c_custkey, weight_string(c_name), weight_string(c_custkey) from customer where c_custkey = :o_custkey group by c_name, c_custkey, weight_string(c_name), weight_string(c_custkey)", + "Table": "customer", + "Values": [ + ":o_custkey" + ], + "Vindex": "hash" + } + ] } ] }