-
Notifications
You must be signed in to change notification settings - Fork 141
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Improve pushdown optimization and logical to physical transformation (#…
…1091) * Add new table scan builder and optimizer rules Signed-off-by: Chen Dai <[email protected]> * Fix jacoco test coverage Signed-off-by: Chen Dai <[email protected]> * Update javadoc with more details Signed-off-by: Chen Dai <[email protected]> * Fix highlight pushdown issue Signed-off-by: Chen Dai <[email protected]> * Rename new class more properly Signed-off-by: Chen Dai <[email protected]> * Fix default sort by doc issue Signed-off-by: Chen Dai <[email protected]> * Rename visit method and javadoc Signed-off-by: Chen Dai <[email protected]> * Move table scan builder and optimize rule to read package Signed-off-by: Chen Dai <[email protected]> * Fix sort push down issue Signed-off-by: Chen Dai <[email protected]> * Move sortByFields to parent scan builder Signed-off-by: Chen Dai <[email protected]> * Add back old test Signed-off-by: Chen Dai <[email protected]> Signed-off-by: Chen Dai <[email protected]>
- Loading branch information
Showing
44 changed files
with
1,810 additions
and
2,050 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
51 changes: 51 additions & 0 deletions
51
.../src/main/java/org/opensearch/sql/planner/optimizer/rule/read/CreateTableScanBuilder.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.optimizer.rule.read; | ||
|
|
||
| import static org.opensearch.sql.planner.optimizer.pattern.Patterns.table; | ||
|
|
||
| import com.facebook.presto.matching.Capture; | ||
| import com.facebook.presto.matching.Captures; | ||
| import com.facebook.presto.matching.Pattern; | ||
| import lombok.Getter; | ||
| import lombok.experimental.Accessors; | ||
| import org.opensearch.sql.planner.logical.LogicalPlan; | ||
| import org.opensearch.sql.planner.logical.LogicalRelation; | ||
| import org.opensearch.sql.planner.optimizer.Rule; | ||
| import org.opensearch.sql.storage.Table; | ||
| import org.opensearch.sql.storage.read.TableScanBuilder; | ||
|
|
||
| /** | ||
| * Rule that replace logical relation operator to {@link TableScanBuilder} for later | ||
| * push down optimization. All push down optimization rules that depends on table scan | ||
| * builder needs to run after this. | ||
| */ | ||
| public class CreateTableScanBuilder implements Rule<LogicalRelation> { | ||
|
|
||
| /** Capture the table inside matched logical relation operator. */ | ||
| private final Capture<Table> capture; | ||
|
|
||
| /** Pattern that matches logical relation operator. */ | ||
| @Accessors(fluent = true) | ||
| @Getter | ||
| private final Pattern<LogicalRelation> pattern; | ||
|
|
||
| /** | ||
| * Construct create table scan builder rule. | ||
| */ | ||
| public CreateTableScanBuilder() { | ||
| this.capture = Capture.newCapture(); | ||
| this.pattern = Pattern.typeOf(LogicalRelation.class) | ||
| .with(table().capturedAs(capture)); | ||
| } | ||
|
|
||
| @Override | ||
| public LogicalPlan apply(LogicalRelation plan, Captures captures) { | ||
| TableScanBuilder scanBuilder = captures.get(capture).createScanBuilder(); | ||
| // TODO: Remove this after Prometheus refactored to new table scan builder too | ||
| return (scanBuilder == null) ? plan : scanBuilder; | ||
| } | ||
| } |
129 changes: 129 additions & 0 deletions
129
core/src/main/java/org/opensearch/sql/planner/optimizer/rule/read/TableScanPushDown.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.optimizer.rule.read; | ||
|
|
||
| import static org.opensearch.sql.planner.optimizer.pattern.Patterns.aggregate; | ||
| import static org.opensearch.sql.planner.optimizer.pattern.Patterns.filter; | ||
| import static org.opensearch.sql.planner.optimizer.pattern.Patterns.highlight; | ||
| import static org.opensearch.sql.planner.optimizer.pattern.Patterns.limit; | ||
| import static org.opensearch.sql.planner.optimizer.pattern.Patterns.project; | ||
| import static org.opensearch.sql.planner.optimizer.pattern.Patterns.scanBuilder; | ||
| import static org.opensearch.sql.planner.optimizer.pattern.Patterns.sort; | ||
| import static org.opensearch.sql.planner.optimizer.rule.read.TableScanPushDown.TableScanPushDownBuilder.match; | ||
|
|
||
| import com.facebook.presto.matching.Capture; | ||
| import com.facebook.presto.matching.Captures; | ||
| import com.facebook.presto.matching.Pattern; | ||
| import com.facebook.presto.matching.pattern.CapturePattern; | ||
| import com.facebook.presto.matching.pattern.WithPattern; | ||
| import java.util.function.BiFunction; | ||
| import org.opensearch.sql.planner.logical.LogicalPlan; | ||
| import org.opensearch.sql.planner.optimizer.Rule; | ||
| import org.opensearch.sql.storage.read.TableScanBuilder; | ||
|
|
||
| /** | ||
| * Rule template for all table scan push down rules. Because all push down optimization rules | ||
| * have similar workflow in common, such as a pattern that match an operator on top of table scan | ||
| * builder, and action that eliminates the original operator if pushed down, this class helps | ||
| * remove redundant code and improve readability. | ||
| * | ||
| * @param <T> logical plan node type | ||
| */ | ||
| public class TableScanPushDown<T extends LogicalPlan> implements Rule<T> { | ||
|
|
||
| /** Push down optimize rule for filtering condition. */ | ||
| public static final Rule<?> PUSH_DOWN_FILTER = | ||
| match( | ||
| filter( | ||
| scanBuilder())) | ||
| .apply((filter, scanBuilder) -> scanBuilder.pushDownFilter(filter)); | ||
|
|
||
| /** Push down optimize rule for aggregate operator. */ | ||
| public static final Rule<?> PUSH_DOWN_AGGREGATION = | ||
| match( | ||
| aggregate( | ||
| scanBuilder())) | ||
| .apply((agg, scanBuilder) -> scanBuilder.pushDownAggregation(agg)); | ||
|
|
||
| /** Push down optimize rule for sort operator. */ | ||
| public static final Rule<?> PUSH_DOWN_SORT = | ||
| match( | ||
| sort( | ||
| scanBuilder())) | ||
| .apply((sort, scanBuilder) -> scanBuilder.pushDownSort(sort)); | ||
|
|
||
| /** Push down optimize rule for limit operator. */ | ||
| public static final Rule<?> PUSH_DOWN_LIMIT = | ||
| match( | ||
| limit( | ||
| scanBuilder())) | ||
| .apply((limit, scanBuilder) -> scanBuilder.pushDownLimit(limit)); | ||
|
|
||
| public static final Rule<?> PUSH_DOWN_PROJECT = | ||
| match( | ||
| project( | ||
| scanBuilder())) | ||
| .apply((project, scanBuilder) -> scanBuilder.pushDownProject(project)); | ||
|
|
||
| public static final Rule<?> PUSH_DOWN_HIGHLIGHT = | ||
| match( | ||
| highlight( | ||
| scanBuilder())) | ||
| .apply((highlight, scanBuilder) -> scanBuilder.pushDownHighlight(highlight)); | ||
|
|
||
|
|
||
| /** Pattern that matches a plan node. */ | ||
| private final WithPattern<T> pattern; | ||
|
|
||
| /** Capture table scan builder inside a plan node. */ | ||
| private final Capture<TableScanBuilder> capture; | ||
|
|
||
| /** Push down function applied to the plan node and captured table scan builder. */ | ||
| private final BiFunction<T, TableScanBuilder, Boolean> pushDownFunction; | ||
|
|
||
|
|
||
| @SuppressWarnings("unchecked") | ||
| private TableScanPushDown(WithPattern<T> pattern, | ||
| BiFunction<T, TableScanBuilder, Boolean> pushDownFunction) { | ||
| this.pattern = pattern; | ||
| this.capture = ((CapturePattern<TableScanBuilder>) pattern.getPattern()).capture(); | ||
| this.pushDownFunction = pushDownFunction; | ||
| } | ||
|
|
||
| @Override | ||
| public Pattern<T> pattern() { | ||
| return pattern; | ||
| } | ||
|
|
||
| @Override | ||
| public LogicalPlan apply(T plan, Captures captures) { | ||
| TableScanBuilder scanBuilder = captures.get(capture); | ||
| if (pushDownFunction.apply(plan, scanBuilder)) { | ||
| return scanBuilder; | ||
| } | ||
| return plan; | ||
| } | ||
|
|
||
| /** | ||
| * Custom builder class other than generated by Lombok to provide more readable code. | ||
| */ | ||
| static class TableScanPushDownBuilder<T extends LogicalPlan> { | ||
|
|
||
| private WithPattern<T> pattern; | ||
|
|
||
| public static <T extends LogicalPlan> | ||
| TableScanPushDownBuilder<T> match(Pattern<T> pattern) { | ||
| TableScanPushDownBuilder<T> builder = new TableScanPushDownBuilder<>(); | ||
| builder.pattern = (WithPattern<T>) pattern; | ||
| return builder; | ||
| } | ||
|
|
||
| public TableScanPushDown<T> apply( | ||
| BiFunction<T, TableScanBuilder, Boolean> pushDownFunction) { | ||
| return new TableScanPushDown<>(pattern, pushDownFunction); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.