fix(interactive): Support Gremlin union Step in Calcite-Based IR La…

…yer (alibaba#3594)

<!--
Thanks for your contribution! please review
https://github.com/alibaba/GraphScope/blob/main/CONTRIBUTING.md before
opening an issue.
-->

## What do these changes do?
as titled.

<!-- Please give a short brief about these changes. -->

## Related issue number

<!-- Are there any issues opened that will be resolved by merging this
change? -->

Fixes

---------

Co-authored-by: bingqing.lbq <[email protected]>

interactive_engine/compiler/ir_experimental_ci.sh

@@ -15,19 +15,6 @@ if [ $exit_code -ne 0 ]; then
     exit 1
 fi
 
-# restart compiler service
-ps -ef | grep "com.alibaba.graphscope.GraphServer" | grep -v grep | awk '{print $2}' | xargs kill -9 || true
-cd ${base_dir} && make run gremlin.script.language.name=antlr_gremlin_calcite &
-sleep 5s
-# run gremlin standard tests to test calcite-based IR layer
-cd ${base_dir} && make gremlin_calcite_test
-exit_code=$?
-# report test result
-if [ $exit_code -ne 0 ]; then
-    echo "ir\(calcite-based\) gremlin integration test on experimental store fail"
-    exit 1
-fi
-
 # restart compiler service
 ps -ef | grep "com.alibaba.graphscope.GraphServer" | grep -v grep | awk '{print $2}' | xargs kill -9 || true
 cd ${base_dir} && make run gremlin.script.language.name=antlr_gremlin_calcite physical.opt.config=proto &

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/CommonOptTable.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.graphscope.common.ir.meta.schema;
+
+import com.alibaba.graphscope.common.ir.tools.Utils;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.plan.RelOptSchema;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelDistribution;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelReferentialConstraint;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.schema.ColumnStrategy;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import java.util.List;
+
+/**
+ * to support common partial computation, we need to wrap the common sub-plan into a {@code RelOptTable}, i.e.
+ * in the query `g.V().out().union(out(), out())`, `g.V().out()` is a common sub-plan, which is denoted as a {@code CommonOptTable} here.
+ */
+public class CommonOptTable implements RelOptTable {
+    private final RelNode common;
+
+    public CommonOptTable(RelNode common) {
+        this.common = common;
+    }
+
+    @Override
+    public List<String> getQualifiedName() {
+        return ImmutableList.of("common#" + this.common.explain().hashCode());
+    }
+
+    public RelNode getCommon() {
+        return common;
+    }
+
+    @Override
+    public RelDataType getRowType() {
+        return Utils.getOutputType(this.common);
+    }
+
+    @Override
+    public @Nullable RelOptSchema getRelOptSchema() {
+        return null;
+    }
+
+    @Override
+    public <C> @Nullable C unwrap(Class<C> clazz) {
+        if (clazz.isInstance(this)) {
+            return clazz.cast(this);
+        }
+        return null;
+    }
+
+    @Override
+    public boolean isKey(ImmutableBitSet immutableBitSet) {
+        throw new UnsupportedOperationException("is key is unsupported yet in statistics");
+    }
+
+    @Override
+    public @Nullable List<ImmutableBitSet> getKeys() {
+        throw new UnsupportedOperationException("get keys is unsupported yet in statistics");
+    }
+
+    @Override
+    public double getRowCount() {
+        throw new UnsupportedOperationException("row count is unsupported yet in statistics");
+    }
+
+    @Override
+    public @Nullable RelDistribution getDistribution() {
+        throw new UnsupportedOperationException("distribution is unsupported yet in statistics");
+    }
+
+    @Override
+    public @Nullable List<RelCollation> getCollationList() {
+        throw new UnsupportedOperationException("collations is unsupported yet in statistics");
+    }
+
+    // not used currently
+
+    @Override
+    public RelNode toRel(ToRelContext toRelContext) {
+        throw new UnsupportedOperationException("toRel is unsupported for it will never be used");
+    }
+
+    @Override
+    public @Nullable List<RelReferentialConstraint> getReferentialConstraints() {
+        throw new UnsupportedOperationException(
+                "referentialConstraints is unsupported for it will never be used");
+    }
+
+    @Override
+    public @Nullable Expression getExpression(Class aClass) {
+        throw new UnsupportedOperationException(
+                "expression is unsupported for it will never be used");
+    }
+
+    @Override
+    public RelOptTable extend(List<RelDataTypeField> list) {
+        throw new UnsupportedOperationException("extend is unsupported for it will never be used");
+    }
+
+    @Override
+    public List<ColumnStrategy> getColumnStrategies() {
+        throw new UnsupportedOperationException(
+                "columnStrategies is unsupported for it will never be used");
+    }
+}

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphHepPlanner.java

@@ -0,0 +1,159 @@
+package com.alibaba.graphscope.common.ir.planner;
+
+import com.alibaba.graphscope.common.ir.meta.schema.CommonOptTable;
+import com.alibaba.graphscope.common.ir.rel.*;
+import com.alibaba.graphscope.common.ir.rel.graph.*;
+import com.alibaba.graphscope.common.ir.rel.graph.match.GraphLogicalMultiMatch;
+import com.alibaba.graphscope.common.ir.rel.graph.match.GraphLogicalSingleMatch;
+
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.logical.LogicalFilter;
+import org.apache.calcite.rel.logical.LogicalUnion;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSubQuery;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import java.util.IdentityHashMap;
+
+/**
+ * Original {@code HepPlanner} skip optimizations to the nested structures, i.e. {@code RelNode} nested in the {@code CommonTableScan} or {@code RexSubQuery},
+ * we supplement this functionality by overriding the {@code findBestExp} method.
+ */
+public class GraphHepPlanner extends HepPlanner {
+    private @Nullable RelNode originalRoot;
+
+    public GraphHepPlanner(HepProgram program) {
+        super(program);
+    }
+
+    @Override
+    public RelNode findBestExp() {
+        return originalRoot.accept(new PlannerVisitor(originalRoot));
+    }
+
+    @Override
+    public void setRoot(RelNode rel) {
+        this.originalRoot = rel;
+    }
+
+    public RelNode findBestExpOfRoot(RelNode root) {
+        super.setRoot(root);
+        return super.findBestExp();
+    }
+
+    private class PlannerVisitor extends GraphShuttle {
+        private final RelNode root;
+        private final IdentityHashMap<RelNode, RelNode> commonRelVisitedMap;
+        // apply optimization to the sub-query
+        private final RexShuttle subQueryPlanner;
+
+        public PlannerVisitor(RelNode root) {
+            this.root = root;
+            this.commonRelVisitedMap = new IdentityHashMap<>();
+            this.subQueryPlanner =
+                    new RexShuttle() {
+                        @Override
+                        public RexNode visitSubQuery(RexSubQuery subQuery) {
+                            RelNode subRel = subQuery.rel;
+                            RelNode newSubRel = subRel.accept(new PlannerVisitor(subRel));
+                            if (newSubRel == subRel) {
+                                return subQuery;
+                            }
+                            return subQuery.clone(newSubRel);
+                        }
+                    };
+        }
+
+        @Override
+        public RelNode visit(GraphLogicalSource source) {
+            return findBestIfRoot(source, source);
+        }
+
+        @Override
+        public RelNode visit(GraphLogicalExpand expand) {
+            return findBestIfRoot(expand, visitChildren(expand));
+        }
+
+        @Override
+        public RelNode visit(GraphLogicalGetV getV) {
+            return findBestIfRoot(getV, visitChildren(getV));
+        }
+
+        @Override
+        public RelNode visit(GraphLogicalPathExpand expand) {
+            return findBestIfRoot(expand, visitChildren(expand));
+        }
+
+        @Override
+        public RelNode visit(GraphLogicalSingleMatch match) {
+            return findBestIfRoot(match, match);
+        }
+
+        @Override
+        public RelNode visit(GraphLogicalMultiMatch match) {
+            return findBestIfRoot(match, match);
+        }
+
+        @Override
+        public RelNode visit(GraphLogicalAggregate aggregate) {
+            return findBestIfRoot(aggregate, visitChildren(aggregate));
+        }
+
+        @Override
+        public RelNode visit(GraphLogicalProject project) {
+            return findBestIfRoot(project, visitChildren(project));
+        }
+
+        @Override
+        public RelNode visit(GraphLogicalSort sort) {
+            return findBestIfRoot(sort, visitChildren(sort));
+        }
+
+        @Override
+        public RelNode visit(GraphPhysicalExpand physicalExpand) {
+            return findBestIfRoot(physicalExpand, visitChildren(physicalExpand));
+        }
+
+        @Override
+        public RelNode visit(GraphPhysicalGetV physicalGetV) {
+            return findBestIfRoot(physicalGetV, visitChildren(physicalGetV));
+        }
+
+        @Override
+        public RelNode visit(LogicalUnion union) {
+            return findBestIfRoot(union, visitChildren(union));
+        }
+
+        @Override
+        public RelNode visit(LogicalFilter filter) {
+            return findBestIfRoot(filter, visitChildren(filter));
+        }
+
+        @Override
+        public RelNode visit(CommonTableScan tableScan) {
+            RelOptTable optTable = tableScan.getTable();
+            if (optTable instanceof CommonOptTable) {
+                RelNode common = ((CommonOptTable) optTable).getCommon();
+                RelNode visited = commonRelVisitedMap.get(common);
+                if (visited == null) {
+                    visited = common.accept(new PlannerVisitor(common));
+                    commonRelVisitedMap.put(common, visited);
+                }
+                return new CommonTableScan(
+                        tableScan.getCluster(),
+                        tableScan.getTraitSet(),
+                        new CommonOptTable(visited));
+            }
+            return tableScan;
+        }
+
+        private RelNode findBestIfRoot(RelNode oldRel, RelNode newRel) {
+            newRel = newRel.accept(this.subQueryPlanner);
+            return oldRel == root ? findBestExpOfRoot(newRel) : newRel;
+        }
+    }
+}

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/CommonTableScan.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.graphscope.common.ir.rel;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelShuttle;
+import org.apache.calcite.rel.core.TableScan;
+
+/**
+ * represent a common sub-plan as a table scan, specific to a {@code CommonOptTable}
+ */
+public class CommonTableScan extends TableScan {
+    public CommonTableScan(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table) {
+        super(cluster, traitSet, ImmutableList.of(), table);
+    }
+
+    @Override
+    public RelNode accept(RelShuttle shuttle) {
+        if (shuttle instanceof GraphShuttle) {
+            return ((GraphShuttle) shuttle).visit(this);
+        }
+        return shuttle.visit(this);
+    }
+}