Skip to content

Commit

Permalink
Legend SQL - support pattern matching (#2746)
Browse files Browse the repository at this point in the history 
* Legend SQL - support pattern matching operators
  • Loading branch information
@gs-jp1
gs-jp1 authored Apr 3, 2024
1 parent 907b2a9 commit 4e21375
Show file tree
Hide file tree
Showing 10 changed files with 189 additions and 41 deletions.
4 changes: 4 additions & 0 deletions .../src/main/antlr4/org/finos/legend/engine/language/sql/grammar/from/antlr4/SqlBaseLexer.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,10 @@ REGEX_MATCH: '~';
REGEX_NO_MATCH: '!~';
REGEX_MATCH_CI: '~*';
REGEX_NO_MATCH_CI: '!~*';
OP_LIKE: '~~';
OP_ILIKE: '~~*';
OP_NOT_LIKE: '!~~';
OP_NOT_ILIKE: '!~~*';

PLUS: '+';
MINUS: '-';
Expand Down
2 changes: 1 addition & 1 deletion ...src/main/antlr4/org/finos/legend/engine/language/sql/grammar/from/antlr4/SqlBaseParser.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ subscriptSafe
;

cmpOp
: EQ | NEQ | LT | LTE | GT | GTE | LLT | REGEX_MATCH | REGEX_NO_MATCH | REGEX_MATCH_CI | REGEX_NO_MATCH_CI
: EQ | NEQ | LT | LTE | GT | GTE | LLT | REGEX_MATCH | REGEX_NO_MATCH | REGEX_MATCH_CI | REGEX_NO_MATCH_CI | OP_LIKE | OP_ILIKE | OP_NOT_LIKE | OP_NOT_ILIKE
;

setCmpQuantifier
Expand Down
16 changes: 16 additions & 0 deletions ...l-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1896,6 +1896,22 @@ private static ComparisonOperator getComparisonOperator(Token symbol)
return ComparisonOperator.GREATER_THAN;
case SqlBaseLexer.GTE:
return ComparisonOperator.GREATER_THAN_OR_EQUAL;
case SqlBaseLexer.REGEX_MATCH:
return ComparisonOperator.REGEX_MATCH;
case SqlBaseLexer.REGEX_MATCH_CI:
return ComparisonOperator.REGEX_MATCH_CI;
case SqlBaseLexer.REGEX_NO_MATCH:
return ComparisonOperator.REGEX_NO_MATCH;
case SqlBaseLexer.REGEX_NO_MATCH_CI:
return ComparisonOperator.REGEX_NO_MATCH_CI;
case SqlBaseLexer.OP_LIKE:
return ComparisonOperator.LIKE;
case SqlBaseLexer.OP_ILIKE:
return ComparisonOperator.ILIKE;
case SqlBaseLexer.OP_NOT_LIKE:
return ComparisonOperator.NOT_LIKE;
case SqlBaseLexer.OP_NOT_ILIKE:
return ComparisonOperator.NOT_ILIKE;
//TODO handle other operators
default:
throw new UnsupportedOperationException("Unsupported operator: " + symbol.getText());
Expand Down
10 changes: 9 additions & 1 deletion ...mar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,15 @@ public class SQLGrammarComposer
Tuples.pair(ComparisonOperator.LESS_THAN_OR_EQUAL, "<="),
Tuples.pair(ComparisonOperator.GREATER_THAN_OR_EQUAL, ">="),
Tuples.pair(ComparisonOperator.IS_DISTINCT_FROM, "IS DISTINCT FROM"),
Tuples.pair(ComparisonOperator.IS_NOT_DISTINCT_FROM, "IS NOT DISTINCT FROM")
Tuples.pair(ComparisonOperator.IS_NOT_DISTINCT_FROM, "IS NOT DISTINCT FROM"),
Tuples.pair(ComparisonOperator.REGEX_MATCH, "~"),
Tuples.pair(ComparisonOperator.REGEX_MATCH_CI, "~*"),
Tuples.pair(ComparisonOperator.REGEX_NO_MATCH, "!~"),
Tuples.pair(ComparisonOperator.REGEX_NO_MATCH_CI, "!~*"),
Tuples.pair(ComparisonOperator.LIKE, "~~"),
Tuples.pair(ComparisonOperator.ILIKE, "~~*"),
Tuples.pair(ComparisonOperator.NOT_LIKE, "!~~"),
Tuples.pair(ComparisonOperator.NOT_ILIKE, "!~~*")
);
private final MutableMap<LogicalBinaryType, String> binaryComparator = UnifiedMap.newMapWith(
Tuples.pair(LogicalBinaryType.AND, "AND"),
Expand Down
13 changes: 13 additions & 0 deletions ...st/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,19 @@ public void testSelectStar()
check("SELECT myTable.* FROM myTable");
}

@Test
public void testPatternMatching()
{
check("SELECT * FROM myTable where 'abc' ~ 'def'");
check("SELECT * FROM myTable where 'abc' ~* 'def'");
check("SELECT * FROM myTable where 'abc' !~ 'def'");
check("SELECT * FROM myTable where 'abc' !~* 'def'");
check("SELECT * FROM myTable where 'abc' ~~ 'def'");
check("SELECT * FROM myTable where 'abc' ~~* 'def'");
check("SELECT * FROM myTable where 'abc' !~~ 'def'");
check("SELECT * FROM myTable where 'abc' !~~* 'def'");
}

@Test
public void testParameters()
{
Expand Down
18 changes: 16 additions & 2 deletions ...ql-postgres-server/src/main/java/org/finos/legend/engine/postgres/TableNameExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,32 @@

public class TableNameExtractor extends SqlBaseParserBaseVisitor<List<QualifiedName>>
{
private final Boolean extractTables;
private final Boolean extractTableFunctions;

public TableNameExtractor()
{
this(true, true);
}

public TableNameExtractor(Boolean extractTables, Boolean extractTableFunctions)
{
this.extractTables = extractTables;
this.extractTableFunctions = extractTableFunctions;
}

@Override
public List<QualifiedName> visitTableName(SqlBaseParser.TableNameContext ctx)
{
QualifiedName qualifiedName = getQualifiedName(ctx.qname());
return Lists.fixedSize.with(qualifiedName);
return this.extractTables ? Lists.fixedSize.with(qualifiedName) : Lists.fixedSize.empty();
}

@Override
public List<QualifiedName> visitTableFunction(SqlBaseParser.TableFunctionContext ctx)
{
QualifiedName qualifiedName = getQualifiedName(ctx.qname());
return Lists.fixedSize.with(qualifiedName);
return this.extractTableFunctions ? Lists.fixedSize.with(qualifiedName) : Lists.fixedSize.empty();
}

@Override
Expand Down
41 changes: 21 additions & 20 deletions ...ostgres-server/src/test/java/org/finos/legend/engine/postgres/TableNameExtractorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,56 +15,57 @@

package org.finos.legend.engine.postgres;

import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.impl.list.mutable.FastList;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.sql.grammar.from.SQLGrammarParser;
import org.finos.legend.engine.language.sql.grammar.from.antlr4.SqlBaseParser;
import org.finos.legend.engine.protocol.sql.metamodel.QualifiedName;
import org.junit.Test;

import java.util.List;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

public class TableNameExtractorTest
{
private static final TableNameExtractor extractor = new TableNameExtractor();


@Test
public void testGetSchemaAndTable()
{
List<QualifiedName> qualifiedNames = getQualifiedNames("SELECT * FROM schema1.table1");
assertEquals(1, qualifiedNames.size());
QualifiedName qualifiedName = Iterables.getOnlyElement(qualifiedNames);
assertEquals(Lists.mutable.of("schema1", "table1"), qualifiedName.parts);
test("SELECT * FROM schema1.table1", Lists.fixedSize.of("schema1.table1"), new TableNameExtractor());
}

@Test
public void testSetQuery()
{
List<QualifiedName> qualifiedNames = getQualifiedNames("SET A=B");
assertEquals(0, qualifiedNames.size());
test("SET A=B", Lists.fixedSize.empty(), new TableNameExtractor());
}

@Test
public void testSelectWithoutTable()
{
List<QualifiedName> qualifiedNames = getQualifiedNames("SELECT 1");
assertEquals(0, qualifiedNames.size());
test("SELECT 1", FastList.newList(), new TableNameExtractor());
}

@Test
public void testFunctionCall()
public void testExtractingDifferentTypes()
{
List<QualifiedName> qualifiedNames = getQualifiedNames("SELECT * FROM service('/my/service')");
assertEquals(1, qualifiedNames.size());
QualifiedName qualifiedName = Iterables.getOnlyElement(qualifiedNames);
assertEquals(Lists.mutable.of("service"), qualifiedName.parts);
test("SELECT * FROM service('/my/service') UNION SELECT * from myTable", Lists.fixedSize.of("service", "myTable"), new TableNameExtractor());
test("SELECT * FROM service('/my/service') UNION SELECT * from myTable", Lists.fixedSize.of("myTable"), new TableNameExtractor(true, false));
test("SELECT * FROM service('/my/service') UNION SELECT * from myTable", Lists.fixedSize.of("service"), new TableNameExtractor(false, true));
test("SELECT * FROM service('/my/service') UNION SELECT * from myTable", Lists.fixedSize.empty(), new TableNameExtractor(false, false));
}

private static List<QualifiedName> getQualifiedNames(String query)
private void test(String sql, List<String> expected, TableNameExtractor extractor)
{
SqlBaseParser parser = SQLGrammarParser.getSqlBaseParser(query, "query");
return parser.singleStatement().accept(extractor);
SqlBaseParser parser = SQLGrammarParser.getSqlBaseParser(sql, "query");
List<QualifiedName> qualifiedNames = parser.singleStatement().accept(extractor);

List<String> result = ListIterate.collect(qualifiedNames, q -> StringUtils.join(q.parts, "."));

assertEquals(expected.size(), result.size());
assertTrue(ListIterate.allSatisfy(expected, result::contains));
}
}
10 changes: 9 additions & 1 deletion ...xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,7 +435,15 @@ Enum meta::external::query::sql::metamodel::ComparisonOperator
GREATER_THAN,
GREATER_THAN_OR_EQUAL,
IS_DISTINCT_FROM,
IS_NOT_DISTINCT_FROM
IS_NOT_DISTINCT_FROM,
REGEX_MATCH,
REGEX_MATCH_CI,
REGEX_NO_MATCH,
REGEX_NO_MATCH_CI,
LIKE,
ILIKE,
NOT_LIKE,
NOT_ILIKE
}

Enum meta::external::query::sql::metamodel::CurrentTimeType
Expand Down
73 changes: 62 additions & 11 deletions ...ine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1401,14 +1401,22 @@ function meta::external::query::sql::transformation::queryToPure::extractNameFro
c:Cast[1] | 'CAST(' + $c.expression->extractNameFromExpression($context) + ' AS ' + $c.type->extractNameFromExpression($context) + ')',
c:ComparisonExpression[1] |
let operator = [
pair(ComparisonOperator.EQUAL, '='),
pair(ComparisonOperator.NOT_EQUAL, '!='),
pair(ComparisonOperator.LESS_THAN, '<'),
pair(ComparisonOperator.LESS_THAN_OR_EQUAL, '<='),
pair(ComparisonOperator.GREATER_THAN, '>'),
pair(ComparisonOperator.GREATER_THAN_OR_EQUAL, '>='),
pair(ComparisonOperator.IS_DISTINCT_FROM, 'IS DISTINCT FROM'),
pair(ComparisonOperator.IS_NOT_DISTINCT_FROM, 'IS NOT DISTINCT FROM')
pair(ComparisonOperator.EQUAL, '='),
pair(ComparisonOperator.NOT_EQUAL, '!='),
pair(ComparisonOperator.LESS_THAN, '<'),
pair(ComparisonOperator.LESS_THAN_OR_EQUAL, '<='),
pair(ComparisonOperator.GREATER_THAN, '>'),
pair(ComparisonOperator.GREATER_THAN_OR_EQUAL,'>='),
pair(ComparisonOperator.IS_DISTINCT_FROM, 'IS DISTINCT FROM'),
pair(ComparisonOperator.IS_NOT_DISTINCT_FROM, 'IS NOT DISTINCT FROM'),
pair(ComparisonOperator.REGEX_MATCH, '~'),
pair(ComparisonOperator.REGEX_MATCH_CI, '~*'),
pair(ComparisonOperator.REGEX_NO_MATCH, '!~'),
pair(ComparisonOperator.REGEX_NO_MATCH_CI, '!~*'),
pair(ComparisonOperator.LIKE, '~~'),
pair(ComparisonOperator.ILIKE, '~~*'),
pair(ComparisonOperator.NOT_LIKE, '!~~'),
pair(ComparisonOperator.NOT_ILIKE, '!~~*')
]->getValue($c.operator);

$c.left->extractNameFromExpression($context) + ' ' + $operator + ' ' + $c.right->extractNameFromExpression($context);,
Expand Down Expand Up @@ -2185,7 +2193,16 @@ function meta::external::query::sql::transformation::queryToPure::functionProces
processor('ltrim', String, {args, fc, ctx | processTrim(ltrim_String_1__String_1_, $args)}),
processor('left', left_String_1__Integer_1__String_1_),
processor('md5', String, {args, fc, ctx | processHash($args, meta::pure::functions::hash::HashType.MD5)}),
processor('regexp_like', matches_String_1__String_1__Boolean_1_),
processor('regexp_like', Boolean, {args, fc, ctx |
assert($args->size() == 2 || $args->size() == 3, 'incorrect number of args to regexp_like');

let caseInsensitive = $args->size() == 3 && $args->at(2)->reactivate()->match([
s:String[1] | $s == 'i',
a:Any[*] | false
]);

createRegexMatch($args->at(0), $args->at(1), $caseInsensitive, false);
}),
processor('repeat', repeatString_String_$0_1$__Integer_1__String_$0_1$_),
processor('replace', replace_String_1__String_1__String_1__String_1_),
processor('reverse', reverseString_String_1__String_1_),
Expand Down Expand Up @@ -3027,11 +3044,45 @@ function <<access.private>> meta::external::query::sql::transformation::queryToP
function <<access.private>> meta::external::query::sql::transformation::queryToPure::processStandardComparison(c:ComparisonExpression[1], left:ValueSpecification[1], right:ValueSpecification[1], type:Type[1], expContext:SqlTransformExpressionContext[1], context:SqlTransformContext[1]):ValueSpecification[1]
{
[
pair(ComparisonOperator.IS_DISTINCT_FROM, | createIsDistinctFrom($left, $right)),
pair(ComparisonOperator.IS_NOT_DISTINCT_FROM, | createIsNotDistinctFrom($left, $right))
pair(ComparisonOperator.IS_DISTINCT_FROM, | createIsDistinctFrom($left, $right)),
pair(ComparisonOperator.IS_NOT_DISTINCT_FROM, | createIsNotDistinctFrom($left, $right)),
pair(ComparisonOperator.REGEX_MATCH, | createRegexMatch($left, $right, false, false)),
pair(ComparisonOperator.REGEX_MATCH_CI, | createRegexMatch($left, $right, true, false)),
pair(ComparisonOperator.REGEX_NO_MATCH, | createRegexMatch($left, $right, false, true)),
pair(ComparisonOperator.REGEX_NO_MATCH_CI, | createRegexMatch($left, $right, true, true)),
pair(ComparisonOperator.LIKE, | createLike($c.left, $c.right, false, false, $expContext, $context)),
pair(ComparisonOperator.ILIKE, | createLike($c.left, $c.right, true, false, $expContext, $context)),
pair(ComparisonOperator.NOT_LIKE, | createLike($c.left, $c.right, false, true, $expContext, $context)),
pair(ComparisonOperator.NOT_ILIKE, | createLike($c.left, $c.right, true, true, $expContext, $context))
]->getValue($c.operator, | createStandardComparison($c, $left, $right, $type))->eval();
}

function <<access.private>> meta::external::query::sql::transformation::queryToPure::createLike(left:meta::external::query::sql::metamodel::Expression[1], right:meta::external::query::sql::metamodel::Expression[1], caseInsensitive:Boolean[1], not:Boolean[1], expContext:SqlTransformExpressionContext[1], context:SqlTransformContext[1]):ValueSpecification[1]
{
let like = ^LikePredicate(value = $left, pattern = $right, ignoreCase = $caseInsensitive);
let expression = if ($not, | ^NotExpression(value = $like), | $like);

processExpression($expression, $expContext, $context);
}

function <<access.private>> meta::external::query::sql::transformation::queryToPure::createRegexMatch(left:ValueSpecification[1], right:ValueSpecification[1], caseInsensitive:Boolean[1], not:Boolean[1]):ValueSpecification[1]
{
assertFalse($caseInsensitive, 'case insensitive regex currently not supported');

let pattern = $right->reactivate()->match([
s:String[1] | $s,
a:Any[*] | fail('regex must be a string literal'); '';
]);

assert($pattern->startsWith('^') && $pattern->endsWith('$'), 'only exact match regex currently supported');

let patternIV = $pattern->substring(1, $pattern->length() - 1)->iv();

let match = nullOrSfe(matches_String_1__String_1__Boolean_1_, [$left, $patternIV]);

if ($not, | nullOrSfe(not_Boolean_1__Boolean_1_, $match), | $match);
}

function <<access.private>> meta::external::query::sql::transformation::queryToPure::processLiteral(literal: Literal[1], expContext:SqlTransformExpressionContext[1], context: SqlTransformContext[1]):ValueSpecification[1]
{
debug('processLiteral', $context.debug);
Expand Down
Loading

0 comments on commit 4e21375

Please sign in to comment.