From 998363dbda2a1645cab84228ed06fabad6b4a0ba Mon Sep 17 00:00:00 2001 From: Hendrik Saly Date: Fri, 10 May 2024 13:02:52 +0200 Subject: [PATCH] PPL lookup prototype --- .../org/opensearch/sql/analysis/Analyzer.java | 99 +++++++++++++ .../sql/ast/AbstractNodeVisitor.java | 5 + .../org/opensearch/sql/ast/dsl/AstDSL.java | 21 +++ .../org/opensearch/sql/ast/tree/Lukk.java | 49 +++++++ .../org/opensearch/sql/executor/Explain.java | 15 ++ .../sql/planner/DefaultImplementor.java | 15 ++ .../sql/planner/logical/LogicalLukk.java | 44 ++++++ .../logical/LogicalPlanNodeVisitor.java | 4 + .../sql/planner/physical/LukkOperator.java | 137 ++++++++++++++++++ .../sql/planner/physical/PhysicalPlanDSL.java | 17 +++ .../physical/PhysicalPlanNodeVisitor.java | 4 + data.sh | 61 ++++++++ .../org/opensearch/sql/ppl/LukkCommandIT.java | 64 ++++++++ .../OpenSearchExecutionProtector.java | 82 +++++++++++ .../plugin/config/OpenSearchPluginModule.java | 4 +- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 14 ++ .../opensearch/sql/ppl/parser/AstBuilder.java | 43 ++++++ .../sql/ppl/utils/ArgumentFactory.java | 8 + .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 14 ++ .../sql/ppl/parser/AstBuilderTest.java | 15 ++ .../ppl/parser/AstExpressionBuilderTest.java | 7 +- .../sql/ppl/utils/ArgumentFactoryTest.java | 52 +++++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 5 + 24 files changed, 775 insertions(+), 6 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/Lukk.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/logical/LogicalLukk.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/LukkOperator.java create mode 100755 data.sh create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/LukkCommandIT.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index d5e8b93b13..aa51d7483f 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -50,6 +50,7 @@ import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.Limit; +import org.opensearch.sql.ast.tree.Lukk; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.Paginate; import org.opensearch.sql.ast.tree.Parse; @@ -66,6 +67,7 @@ import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; @@ -89,6 +91,7 @@ import org.opensearch.sql.planner.logical.LogicalFetchCursor; import org.opensearch.sql.planner.logical.LogicalFilter; import org.opensearch.sql.planner.logical.LogicalLimit; +import org.opensearch.sql.planner.logical.LogicalLukk; import org.opensearch.sql.planner.logical.LogicalML; import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPaginate; @@ -507,6 +510,102 @@ public LogicalPlan visitDedupe(Dedupe node, AnalysisContext context) { consecutive); } + /** Build {@link LogicalLukk}. */ + @Override + public LogicalPlan visitLukk(Lukk node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + List options = node.getOptions(); + // Todo, refactor the option. + Boolean appendOnly = (Boolean) options.get(0).getValue().getValue(); + + Table table = + dataSourceService + .getDataSource(DEFAULT_DATASOURCE_NAME) + .getStorageEngine() + .getTable(null, node.getIndexName()); + + if (table == null || !table.exists()) { + throw new SemanticCheckException( + String.format("no such lookup index %s", node.getIndexName())); + } + + return new LogicalLukk( + child, + node.getIndexName(), + analyzeLukkMatchFields(node.getMatchFieldList(), context), + appendOnly, + analyzeLukkCopyFields(node.getCopyFieldList(), context, table)); + } + + private ImmutableMap analyzeLukkMatchFields( + List inputMap, AnalysisContext context) { + ImmutableMap.Builder copyMapBuilder = + new ImmutableMap.Builder<>(); + for (Map resultMap : inputMap) { + Expression origin = expressionAnalyzer.analyze(resultMap.getOrigin(), context); + if (resultMap.getTarget() instanceof Field) { + ReferenceExpression target = + new ReferenceExpression( + ((Field) resultMap.getTarget()).getField().toString(), origin.type()); + ReferenceExpression originExpr = DSL.ref(origin.toString(), origin.type()); + TypeEnvironment curEnv = context.peek(); + curEnv.remove(originExpr); + curEnv.define(target); + copyMapBuilder.put(originExpr, target); + } else { + throw new SemanticCheckException( + String.format("the target expected to be field, but is %s", resultMap.getTarget())); + } + } + + return copyMapBuilder.build(); + } + + private ImmutableMap analyzeLukkCopyFields( + List inputMap, AnalysisContext context, Table table) { + + TypeEnvironment curEnv = context.peek(); + java.util.Map fieldTypes = table.getFieldTypes(); + + if (inputMap.isEmpty()) { + fieldTypes.forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v)); + return ImmutableMap.builder().build(); + } + + ImmutableMap.Builder copyMapBuilder = + new ImmutableMap.Builder<>(); + for (Map resultMap : inputMap) { + if (resultMap.getOrigin() instanceof Field && resultMap.getTarget() instanceof Field) { + String fieldName = ((Field) resultMap.getOrigin()).getField().toString(); + ExprType ex = fieldTypes.get(fieldName); + + if (ex == null) { + throw new SemanticCheckException(String.format("no such field %s", fieldName)); + } + + ReferenceExpression origin = new ReferenceExpression(fieldName, ex); + + if (resultMap.getTarget().equals(resultMap.getOrigin())) { + + curEnv.define(origin); + copyMapBuilder.put(origin, origin); + } else { + ReferenceExpression target = + new ReferenceExpression(((Field) resultMap.getTarget()).getField().toString(), ex); + curEnv.define(target); + copyMapBuilder.put(origin, target); + } + } else { + throw new SemanticCheckException( + String.format( + "the origin and target expected to be field, but is %s/%s", + resultMap.getOrigin(), resultMap.getTarget())); + } + } + + return copyMapBuilder.build(); + } + /** Logical head is identical to {@link LogicalLimit}. */ public LogicalPlan visitHead(Head node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 973b10310b..7a901771e8 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -49,6 +49,7 @@ import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.Limit; +import org.opensearch.sql.ast.tree.Lukk; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.Paginate; import org.opensearch.sql.ast.tree.Parse; @@ -217,6 +218,10 @@ public T visitDedupe(Dedupe node, C context) { return visitChildren(node, context); } + public T visitLukk(Lukk node, C context) { + return visitChildren(node, context); + } + public T visitHead(Head node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 4f3056b0f7..0ac3d4b031 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ast.dsl; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -49,6 +50,7 @@ import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Limit; +import org.opensearch.sql.ast.tree.Lukk; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.RareTopN; @@ -441,6 +443,25 @@ public static Dedupe dedupe(UnresolvedPlan input, List options, Field. return new Dedupe(input, options, Arrays.asList(fields)); } + public static Lukk lukk( + UnresolvedPlan input, + String indexName, + List matchFieldList, + List options, + List copyFieldList) { + return new Lukk(input, indexName, matchFieldList, options, copyFieldList); + } + + public static List fieldMap(String field, String asField, String... more) { + assert more == null || more.length % 2 == 0; + List list = new ArrayList(); + list.add(map(field, asField)); + for (int i = 0; i < more.length; i = i + 2) { + list.add(map(more[i], more[i + 1])); + } + return list; + } + public static Head head(UnresolvedPlan input, Integer size, Integer from) { return new Head(input, size, from); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Lukk.java b/core/src/main/java/org/opensearch/sql/ast/tree/Lukk.java new file mode 100644 index 0000000000..b066b4c8d3 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Lukk.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.ast.expression.Map; + +/** AST node represent Lukk operation. */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +@AllArgsConstructor +public class Lukk extends UnresolvedPlan { + private UnresolvedPlan child; + private final String indexName; + private final List matchFieldList; + private final List options; + private final List copyFieldList; + + @Override + public Lukk attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitLukk(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/Explain.java b/core/src/main/java/org/opensearch/sql/executor/Explain.java index 0f05b99383..79e44b513f 100644 --- a/core/src/main/java/org/opensearch/sql/executor/Explain.java +++ b/core/src/main/java/org/opensearch/sql/executor/Explain.java @@ -22,6 +22,7 @@ import org.opensearch.sql.planner.physical.EvalOperator; import org.opensearch.sql.planner.physical.FilterOperator; import org.opensearch.sql.planner.physical.LimitOperator; +import org.opensearch.sql.planner.physical.LukkOperator; import org.opensearch.sql.planner.physical.NestedOperator; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlanNodeVisitor; @@ -157,6 +158,20 @@ public ExplainResponseNode visitDedupe(DedupeOperator node, Object context) { "consecutive", node.getConsecutive()))); } + @Override + public ExplainResponseNode visitLukk(LukkOperator node, Object context) { + return explain( + node, + context, + explainNode -> + explainNode.setDescription( + ImmutableMap.of( + "copyfields", node.getCopyFieldMap(), + "matchfields", node.getMatchFieldMap(), + "indexname", node.getIndexName(), + "appendonly", node.getAppendOnly()))); + } + @Override public ExplainResponseNode visitRareTopN(RareTopNOperator node, Object context) { return explain( diff --git a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java index b53d17b38f..c08e219f7a 100644 --- a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java @@ -13,6 +13,7 @@ import org.opensearch.sql.planner.logical.LogicalFetchCursor; import org.opensearch.sql.planner.logical.LogicalFilter; import org.opensearch.sql.planner.logical.LogicalLimit; +import org.opensearch.sql.planner.logical.LogicalLukk; import org.opensearch.sql.planner.logical.LogicalNested; import org.opensearch.sql.planner.logical.LogicalPaginate; import org.opensearch.sql.planner.logical.LogicalPlan; @@ -31,6 +32,7 @@ import org.opensearch.sql.planner.physical.EvalOperator; import org.opensearch.sql.planner.physical.FilterOperator; import org.opensearch.sql.planner.physical.LimitOperator; +import org.opensearch.sql.planner.physical.LukkOperator; import org.opensearch.sql.planner.physical.NestedOperator; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.ProjectOperator; @@ -74,6 +76,19 @@ public PhysicalPlan visitDedupe(LogicalDedupe node, C context) { node.getConsecutive()); } + @Override + public PhysicalPlan visitLukk(LogicalLukk node, C context) { + return new LukkOperator( + visitChild(node, context), + node.getIndexName(), + node.getMatchFieldMap(), + node.getAppendOnly(), + node.getCopyFieldMap(), + (a, b) -> { + throw new RuntimeException("not implemented by DefaultImplementor"); + }); + } + @Override public PhysicalPlan visitProject(LogicalProject node, C context) { return new ProjectOperator( diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalLukk.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalLukk.java new file mode 100644 index 0000000000..2638db221e --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalLukk.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.logical; + +import java.util.Arrays; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.expression.ReferenceExpression; + +/** Logical Dedupe Plan. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class LogicalLukk extends LogicalPlan { + + private final String indexName; + private final Map matchFieldMap; + private final Map copyFieldMap; + private final Boolean appendOnly; + + /** Constructor of LogicalDedupe. */ + public LogicalLukk( + LogicalPlan child, + String indexName, + Map matchFieldMap, + Boolean appendOnly, + Map copyFieldMap) { + super(Arrays.asList(child)); + this.indexName = indexName; + this.copyFieldMap = copyFieldMap; + this.matchFieldMap = matchFieldMap; + this.appendOnly = appendOnly; + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitLukk(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java index 156db35306..a3201508eb 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -52,6 +52,10 @@ public R visitDedupe(LogicalDedupe plan, C context) { return visitNode(plan, context); } + public R visitLukk(LogicalLukk plan, C context) { + return visitNode(plan, context); + } + public R visitRename(LogicalRename plan, C context) { return visitNode(plan, context); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/LukkOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/LukkOperator.java new file mode 100644 index 0000000000..2058515743 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/LukkOperator.java @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.physical; + +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NonNull; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.expression.ReferenceExpression; + +/** Lukk operator. Perform lookup on another OpenSearch index and enrich the results. */ +@Getter +@EqualsAndHashCode(callSuper = false) +public class LukkOperator extends PhysicalPlan { + @Getter private final PhysicalPlan input; + @Getter private final String indexName; + @Getter private final Map matchFieldMap; + @Getter private final Map copyFieldMap; + @Getter private final Boolean appendOnly; + private final BiFunction, Map> lookup; + + /** Lukk Constructor. */ + @NonNull + public LukkOperator( + PhysicalPlan input, + String indexName, + Map matchFieldMap, + Boolean appendOnly, + Map copyFieldMap, + BiFunction, Map> lookup) { + this.input = input; + this.indexName = indexName; + this.matchFieldMap = matchFieldMap; + this.appendOnly = appendOnly; + this.copyFieldMap = copyFieldMap; + this.lookup = lookup; + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitLukk(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return input.hasNext(); + } + + @Override + public ExprValue next() { + ExprValue inputValue = input.next(); + + if (STRUCT == inputValue.type()) { + Map matchMap = new HashMap<>(); + Map finalMap = new HashMap<>(); + + for (Map.Entry matchField : + matchFieldMap.entrySet()) { + Object val = inputValue.bindingTuples().resolve(matchField.getValue()).value(); + if (val != null) { + matchMap.put(matchField.getKey().toString(), val); + } else { + // System.out.println("No value in search results for " + matchField.getValue() + " + // field"); + // No value in search results, so we just return the input + return inputValue; + } + } + + finalMap.put("_match", matchMap); + + Map copyMap = new HashMap<>(); + + if (!copyFieldMap.isEmpty()) { + + for (Map.Entry copyField : + copyFieldMap.entrySet()) { + copyMap.put(String.valueOf(copyField.getKey()), String.valueOf(copyField.getValue())); + } + + finalMap.put("_copy", copyMap.keySet()); + } + + Map source = lookup.apply(indexName, finalMap); + + if (source == null || source.isEmpty()) { + // no lookup found or lookup is empty, so we just return the original input value + return inputValue; + } + + Map tupleValue = ExprValueUtils.getTupleValue(inputValue); + Map resultBuilder = new HashMap<>(); + resultBuilder.putAll(tupleValue); + + if (appendOnly) { + + for (Map.Entry sourceField : source.entrySet()) { + String u = copyMap.get(sourceField.getKey()); + resultBuilder.putIfAbsent( + u == null ? sourceField.getKey() : u.toString(), + ExprValueUtils.fromObjectValue(sourceField.getValue())); + } + } else { + // default + + for (Map.Entry sourceField : source.entrySet()) { + String u = copyMap.get(sourceField.getKey()); + resultBuilder.put( + u == null ? sourceField.getKey() : u.toString(), + ExprValueUtils.fromObjectValue(sourceField.getValue())); + } + } + + return ExprTupleValue.fromExprValueMap(resultBuilder); + + } else { + return inputValue; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java index 147f0e08dc..b7ecda48ea 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java @@ -78,6 +78,23 @@ public static DedupeOperator dedupe( input, Arrays.asList(expressions), allowedDuplication, keepEmpty, consecutive); } + public static LukkOperator lukk( + PhysicalPlan input, + String indexName, + Map matchFieldMap, + Boolean appendOnly, + Map copyFieldMap) { + return new LukkOperator( + input, + indexName, + matchFieldMap, + appendOnly, + copyFieldMap, + (a, b) -> { + throw new RuntimeException("not implemented by PhysicalPlanDSL"); + }); + } + public WindowOperator window( PhysicalPlan input, NamedExpression windowFunction, WindowDefinition windowDefinition) { return new WindowOperator(input, windowFunction, windowDefinition); diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 99b5cc8020..a1fd8d0fca 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -64,6 +64,10 @@ public R visitDedupe(DedupeOperator node, C context) { return visitNode(node, context); } + public R visitLukk(LukkOperator node, C context) { + return visitNode(node, context); + } + public R visitValues(ValuesOperator node, C context) { return visitNode(node, context); } diff --git a/data.sh b/data.sh new file mode 100755 index 0000000000..6c932cd26e --- /dev/null +++ b/data.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -e +./gradlew assemble +~/Downloads/opensearch-3.0.0-SNAPSHOT/bin/opensearch-plugin remove opensearch-sql + +~/Downloads/opensearch-3.0.0-SNAPSHOT/bin/opensearch-plugin install -b file:///Users/salyh/devel/eliatra/awsppl/sql/plugin/build/distributions/opensearch-sql-3.0.0.0-SNAPSHOT.zip + +exit + +curl localhost:9200/users/_doc -H 'content-type: application/json' -d ' +{ +"name": "Hendrik", +"department": "IT", +"uid": 1981 +} +' + +curl localhost:9200/users/_doc -H 'content-type: application/json' -d ' +{ +"name": "David", +"department": "Management", +"uid": 76 +} +' + +curl localhost:9200/users/_doc -H 'content-type: application/json' -d ' +{ +"name": "Bruce", +"department": "Sales", +"uid": 788 +} +' + +curl localhost:9200/logins/_doc -H 'content-type: application/json' -d ' +{ +"date": "2020-08-17 14:09:00 UTC", +"uid": 788 +} +' + +curl localhost:9200/logins/_doc -H 'content-type: application/json' -d ' +{ +"date": "2020-08-17 16:11:00 UTC", +"uid": 0 +} +' + +curl localhost:9200/logins/_doc -H 'content-type: application/json' -d ' +{ +"date": "2020-08-17 19:19:01 UTC", +"name": "Hendrik" +} +' + +curl localhost:9200/logins/_doc -H 'content-type: application/json' -d ' +{ +"date": "2020-08-18 22:13:01 UTC", +"name": "Bruce", +"department": "Sales" +} +' \ No newline at end of file diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/LukkCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/LukkCommandIT.java new file mode 100644 index 0000000000..e0588238af --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/LukkCommandIT.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class LukkCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.BANK); + loadIndex(Index.BANK_WITH_NULL_VALUES); + } + + @Test + public void testLukk() throws IOException { + JSONObject result = executeQuery(String.format("source=%s | lukk %s male", TEST_INDEX_BANK)); + verifyDataRows(result, rows(true), rows(false)); + } + + @Test + public void testConsecutiveDedup() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | dedup male consecutive=true | fields male", TEST_INDEX_BANK)); + verifyDataRows(result, rows(true), rows(false), rows(true), rows(false)); + } + + @Test + public void testAllowMoreDuplicates() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | dedup 2 male | fields male", TEST_INDEX_BANK)); + verifyDataRows(result, rows(true), rows(true), rows(false), rows(false)); + } + + @Test + public void testKeepEmptyDedup() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | dedup balance keepempty=true | fields firstname, balance", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifyDataRows( + result, + rows("Amber JOHnny", 39225), + rows("Hattie", null), + rows("Nanette", 32838), + rows("Dale", 4180), + rows("Elinor", null), + rows("Virginia", null), + rows("Dillard", 48086)); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java index 0905c2f4b4..158c17b602 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java @@ -5,8 +5,20 @@ package org.opensearch.sql.opensearch.executor.protector; +import java.util.Map; +import java.util.Set; +import java.util.function.BiFunction; +import javax.annotation.Nullable; +import lombok.AllArgsConstructor; import lombok.RequiredArgsConstructor; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.sql.monitor.ResourceMonitor; +import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.planner.physical.ADOperator; import org.opensearch.sql.opensearch.planner.physical.MLCommonsOperator; import org.opensearch.sql.opensearch.planner.physical.MLOperator; @@ -16,6 +28,7 @@ import org.opensearch.sql.planner.physical.EvalOperator; import org.opensearch.sql.planner.physical.FilterOperator; import org.opensearch.sql.planner.physical.LimitOperator; +import org.opensearch.sql.planner.physical.LukkOperator; import org.opensearch.sql.planner.physical.NestedOperator; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.ProjectOperator; @@ -29,11 +42,16 @@ /** OpenSearch Execution Protector. */ @RequiredArgsConstructor +@AllArgsConstructor public class OpenSearchExecutionProtector extends ExecutionProtector { /** OpenSearch resource monitor. */ private final ResourceMonitor resourceMonitor; + @Nullable + /** OpenSearch client. Maybe null * */ + private OpenSearchClient openSearchClient; + public PhysicalPlan protect(PhysicalPlan physicalPlan) { return physicalPlan.accept(this, null); } @@ -116,6 +134,70 @@ public PhysicalPlan visitDedupe(DedupeOperator node, Object context) { node.getConsecutive()); } + @Override + public PhysicalPlan visitLukk(LukkOperator node, Object context) { + return new LukkOperator( + visitInput(node.getInput(), context), + node.getIndexName(), + node.getMatchFieldMap(), + node.getAppendOnly(), + node.getCopyFieldMap(), + lookup()); + } + + private BiFunction, Map> lookup() { + + if (openSearchClient == null) { + throw new RuntimeException( + "Can not perform lookup because openSearchClient was null. This is likely a bug."); + } + + return (indexName, inputMap) -> { + Map matchMap = (Map) inputMap.get("_match"); + Set copySet = (Set) inputMap.get("_copy"); + + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + + for (Map.Entry f : matchMap.entrySet()) { + BoolQueryBuilder orQueryBuilder = new BoolQueryBuilder(); + + // Todo: Search with term and a match query? Or terms only? + orQueryBuilder.should(new TermQueryBuilder(f.getKey(), f.getValue().toString())); + orQueryBuilder.should(new MatchQueryBuilder(f.getKey(), f.getValue().toString())); + orQueryBuilder.minimumShouldMatch(1); + + // filter is the same as "must" but ignores scoring + boolQueryBuilder.filter(orQueryBuilder); + } + + SearchResponse result = + openSearchClient + .getNodeClient() + .search( + new SearchRequest(indexName) + .source( + SearchSourceBuilder.searchSource() + .fetchSource( + copySet == null ? null : copySet.toArray(new String[0]), null) + .query(boolQueryBuilder) + .size(2))) + .actionGet(); + + int hits = result.getHits().getHits().length; + + if (hits == 0) { + // null indicates no hits for the lookup found + return null; + } + + if (hits != 1) { + throw new RuntimeException("too many hits for " + indexName + " (" + hits + ")"); + } + + return result.getHits().getHits()[0].getSourceAsMap(); + }; + } + @Override public PhysicalPlan visitWindow(WindowOperator node, Object context) { return new WindowOperator( diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java index 33a785c498..8789c79c30 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java @@ -69,8 +69,8 @@ public ResourceMonitor resourceMonitor(Settings settings) { } @Provides - public ExecutionProtector protector(ResourceMonitor resourceMonitor) { - return new OpenSearchExecutionProtector(resourceMonitor); + public ExecutionProtector protector(ResourceMonitor resourceMonitor, OpenSearchClient client) { + return new OpenSearchExecutionProtector(resourceMonitor, client); } @Provides diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 9f707c13cd..833a57fdb4 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -35,6 +35,7 @@ NEW_FIELD: 'NEW_FIELD'; KMEANS: 'KMEANS'; AD: 'AD'; ML: 'ML'; +LUKK: 'LUKK'; // COMMAND ASSIST KEYWORDS AS: 'AS'; @@ -57,6 +58,7 @@ NUM: 'NUM'; // ARGUMENT KEYWORDS KEEPEMPTY: 'KEEPEMPTY'; CONSECUTIVE: 'CONSECUTIVE'; +APPENDONLY: 'APPENDONLY'; DEDUP_SPLITVALUES: 'DEDUP_SPLITVALUES'; PARTITIONS: 'PARTITIONS'; ALLNUM: 'ALLNUM'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 5a9c179d1a..864cead088 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -38,6 +38,7 @@ commands | renameCommand | statsCommand | dedupCommand + | lukkCommand | sortCommand | evalCommand | headCommand @@ -85,6 +86,18 @@ dedupCommand : DEDUP (number = integerLiteral)? fieldList (KEEPEMPTY EQUAL keepempty = booleanLiteral)? (CONSECUTIVE EQUAL consecutive = booleanLiteral)? ; +matchFieldWithOptAs + : orignalMatchField = fieldExpression (AS asMatchField = fieldExpression)? + ; + +copyFieldWithOptAs + : orignalCopyField = fieldExpression (AS asCopyField = fieldExpression)? + ; + +lukkCommand + : LUKK tableSource matchFieldWithOptAs (COMMA matchFieldWithOptAs)* (APPENDONLY EQUAL appendonly = booleanLiteral)? (copyFieldWithOptAs (COMMA copyFieldWithOptAs)*)* + ; + sortCommand : SORT sortbyClause ; @@ -832,6 +845,7 @@ keywordsCanBeId | RENAME | STATS | DEDUP + | LUKK | SORT | EVAL | HEAD diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 3c693fa0bd..6642fccb13 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldsCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FromClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.HeadCommandContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LukkCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RareCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RenameCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SearchFilterFromContext; @@ -54,6 +55,7 @@ import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Kmeans; +import org.opensearch.sql.ast.tree.Lukk; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; @@ -212,6 +214,47 @@ public UnresolvedPlan visitDedupCommand(DedupCommandContext ctx) { return new Dedupe(ArgumentFactory.getArgumentList(ctx), getFieldList(ctx.fieldList())); } + /** Lookup command */ + @Override + public UnresolvedPlan visitLukkCommand(LukkCommandContext ctx) { + ArgumentFactory.getArgumentList(ctx); + ctx.tableSource(); + ctx.copyFieldWithOptAs(); + ctx.matchFieldWithOptAs(); + return new Lukk( + ctx.tableSource().tableQualifiedName().getText(), + ctx.matchFieldWithOptAs().stream() + .map( + ct -> + new Map( + evaluateFieldExpressionContext(ct.orignalMatchField), + evaluateFieldExpressionContext(ct.asMatchField, ct.orignalMatchField))) + .collect(Collectors.toList()), + ArgumentFactory.getArgumentList(ctx), + ctx.copyFieldWithOptAs().stream() + .map( + ct -> + new Map( + evaluateFieldExpressionContext(ct.orignalCopyField), + evaluateFieldExpressionContext(ct.asCopyField, ct.orignalCopyField))) + .collect(Collectors.toList())); + } + + private UnresolvedExpression evaluateFieldExpressionContext( + OpenSearchPPLParser.FieldExpressionContext f) { + return internalVisitExpression(f); + } + + private UnresolvedExpression evaluateFieldExpressionContext( + OpenSearchPPLParser.FieldExpressionContext f0, + OpenSearchPPLParser.FieldExpressionContext f1) { + if (f0 == null) { + return internalVisitExpression(f1); + } else { + return internalVisitExpression(f0); + } + } + /** Head command visitor. */ @Override public UnresolvedPlan visitHeadCommand(HeadCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index f89ecf9c6e..50379929da 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DedupCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldsCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IntegerLiteralContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LukkCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RareCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SortFieldContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.StatsCommandContext; @@ -80,6 +81,13 @@ public static List getArgumentList(DedupCommandContext ctx) { : new Argument("consecutive", new Literal(false, DataType.BOOLEAN))); } + public static List getArgumentList(LukkCommandContext ctx) { + return Arrays.asList( + ctx.appendonly != null + ? new Argument("appendonly", getArgumentValue(ctx.appendonly)) + : new Argument("appendonly", new Literal(false, DataType.BOOLEAN))); + } + /** * Get list of {@link Argument}. * diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index d28e5d122b..ff5aae500b 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -213,6 +213,20 @@ public String visitDedupe(Dedupe node, String context) { child, fields, allowedDuplication, keepEmpty, consecutive); } + // Todo: do we need anonymization for lookups? + /* + @Override + public String visitLukk(Lukk node, String context) { + String child = node.getChild().get(0).accept(this, context); + String fields = visitFieldList(node.getFields()); + List options = node.getOptions(); + Boolean appendonly = (Boolean) options.get(0).getValue().getValue(); + + return StringUtils.format( + "%s | lukk %s %d appendonly=%b ...", + child, fields, appendonly); + }*/ + @Override public String visitHead(Head node, String context) { String child = node.getChild().get(0).accept(this, context); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index c9989a49c4..47cf3585a4 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -21,11 +21,13 @@ import static org.opensearch.sql.ast.dsl.AstDSL.eval; import static org.opensearch.sql.ast.dsl.AstDSL.exprList; import static org.opensearch.sql.ast.dsl.AstDSL.field; +import static org.opensearch.sql.ast.dsl.AstDSL.fieldMap; import static org.opensearch.sql.ast.dsl.AstDSL.filter; import static org.opensearch.sql.ast.dsl.AstDSL.function; import static org.opensearch.sql.ast.dsl.AstDSL.head; import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.let; +import static org.opensearch.sql.ast.dsl.AstDSL.lukk; import static org.opensearch.sql.ast.dsl.AstDSL.map; import static org.opensearch.sql.ast.dsl.AstDSL.nullLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.parse; @@ -44,6 +46,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Arrays; +import java.util.Collections; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -378,6 +381,18 @@ public void testDedupCommandWithSortby() { defaultDedupArgs())); } + @Test + public void testLukkCommand() { + assertEqual( + "source=t | lukk a field", + lukk( + relation("t"), + "a", + fieldMap("field", "field"), + exprList(argument("appendonly", booleanLiteral(false))), + Collections.emptyList())); + } + @Test public void testHeadCommand() { assertEqual("source=t | head", head(relation("t"), 10, 0)); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 7bcb87d193..f2e1b7c9f2 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -616,10 +616,9 @@ public void functionNameCanBeUsedAsIdentifier() { + " | LOG10 | LOG2 | MOD | PI |POW | POWER | RAND | ROUND | SIGN | SQRT | TRUNCATE " + "| ACOS | ASIN | ATAN | ATAN2 | COS | COT | DEGREES | RADIANS | SIN | TAN"); assertFunctionNameCouldBeId( - "SEARCH | DESCRIBE | SHOW | FROM | WHERE | FIELDS | RENAME | STATS " - + "| DEDUP | SORT | EVAL | HEAD | TOP | RARE | PARSE | METHOD | REGEX | PUNCT | GROK " - + "| PATTERN | PATTERNS | NEW_FIELD | KMEANS | AD | ML | SOURCE | INDEX | D | DESC " - + "| DATASOURCES"); + "SEARCH | DESCRIBE | SHOW | FROM | WHERE | FIELDS | RENAME | STATS | DEDUP | LUKK | SORT |" + + " EVAL | HEAD | TOP | RARE | PARSE | METHOD | REGEX | PUNCT | GROK | PATTERN |" + + " PATTERNS | NEW_FIELD | KMEANS | AD | ML | SOURCE | INDEX | D | DESC | DATASOURCES"); } void assertFunctionNameCouldBeId(String antlrFunctionName) { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java index 761dbe2997..c4c15fb992 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java @@ -14,12 +14,15 @@ import static org.opensearch.sql.ast.dsl.AstDSL.dedupe; import static org.opensearch.sql.ast.dsl.AstDSL.exprList; import static org.opensearch.sql.ast.dsl.AstDSL.field; +import static org.opensearch.sql.ast.dsl.AstDSL.fieldMap; import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.lukk; import static org.opensearch.sql.ast.dsl.AstDSL.projectWithArg; import static org.opensearch.sql.ast.dsl.AstDSL.relation; import static org.opensearch.sql.ast.dsl.AstDSL.sort; import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; +import java.util.Collections; import org.junit.Test; import org.opensearch.sql.ppl.parser.AstBuilderTest; @@ -102,4 +105,53 @@ public void testSortFieldArgument() { public void testNoArgConstructorForArgumentFactoryShouldPass() { new ArgumentFactory(); } + + @Test + public void testLukkCommandRequiredArguments() { + assertEqual( + "source=t | lukk a field", + lukk( + relation("t"), + "a", + fieldMap("field", "field"), + exprList(argument("appendonly", booleanLiteral(false))), + Collections.emptyList())); + } + + @Test + public void testLukkCommandFieldArguments() { + assertEqual( + "source=t | lukk a field AS field1,field2 AS field3 destfield AS destfield1, destfield2 AS" + + " destfield3", + lukk( + relation("t"), + "a", + fieldMap("field", "field1", "field2", "field3"), + exprList(argument("appendonly", booleanLiteral(false))), + fieldMap("destfield", "destfield1", "destfield2", "destfield3"))); + } + + @Test + public void testLukkCommandAppendTrueArgument() { + assertEqual( + "source=t | lukk a field appendonly=true", + lukk( + relation("t"), + "a", + fieldMap("field", "field"), + exprList(argument("appendonly", booleanLiteral(true))), + Collections.emptyList())); + } + + @Test + public void testLukkCommandAppendFalseArgument() { + assertEqual( + "source=t | lukk a field appendonly=false", + lukk( + relation("t"), + "a", + fieldMap("field", "field"), + exprList(argument("appendonly", booleanLiteral(false))), + Collections.emptyList())); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index cd51ea07df..8cb8f86ba7 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -89,6 +89,11 @@ public void testDedupCommand() { anonymize("source=t | dedup f1, f2")); } + @Test + public void testLukkCommand() { + assertEquals("source=t | lukk ", anonymize("source=t | lukk a field1, field2")); + } + @Test public void testHeadCommandWithNumber() { assertEquals("source=t | head 3", anonymize("source=t | head 3"));