feat(hogql): Allow a placeholder to be used in place of a select stat…

…ement (#19767) * Allow a placeholder to be used in place of a select statement in a union all * Updated the hogql parser version * Use new hogql-parser version * Fixed tests * Update query snapshots * Update query snapshots * Rejigged the g4 file * Updated mypy * Updated mypy --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
PostHog · Jan 18, 2024 · d946f66 · d946f66
1 parent 9104b58
commit d946f66
Show file tree

Hide file tree

Showing 15 changed files with 1,967 additions and 1,933 deletions.
diff --git a/hogql_parser/HogQLParser.cpp b/hogql_parser/HogQLParser.cpp
diff --git a/hogql_parser/HogQLParser.h b/hogql_parser/HogQLParser.h
@@ -204,6 +204,7 @@ class  HogQLParser : public antlr4::Parser {
     antlr4::tree::TerminalNode *LPAREN();
     SelectUnionStmtContext *selectUnionStmt();
     antlr4::tree::TerminalNode *RPAREN();
+    PlaceholderContext *placeholder();
 
 
     virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;

diff --git a/hogql_parser/HogQLParser.interp b/hogql_parser/HogQLParser.interp
diff --git a/hogql_parser/parser.cpp b/hogql_parser/parser.cpp
@@ -310,6 +310,12 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor {
     if (select_stmt_ctx) {
       return visit(select_stmt_ctx);
     }
+
+    auto placeholder_ctx = ctx->placeholder();
+    if (placeholder_ctx) {
+      return visitAsPyObject(placeholder_ctx);
+    }
+
     return visit(ctx->selectUnionStmt());
   }
 
@@ -344,6 +350,9 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor {
         int extend_code = X_PyList_Extend(flattened_queries, sub_select_queries);
         if (extend_code == -1) goto select_queries_loop_py_error;
         Py_DECREF(sub_select_queries);
+      } else if (is_ast_node_instance(query, "Placeholder")) {
+        int append_code = PyList_Append(flattened_queries, query);
+        if (append_code == -1) goto select_queries_loop_py_error;
       } else {
         Py_DECREF(flattened_queries);
         X_Py_DECREF_ALL(select_queries);

diff --git a/hogql_parser/setup.py b/hogql_parser/setup.py
@@ -32,7 +32,7 @@
 
 setup(
     name="hogql_parser",
-    version="1.0.2",
+    version="1.0.3",
     url="https://github.com/PostHog/posthog/tree/master/hogql_parser",
     author="PostHog Inc.",
     author_email="[email protected]",

diff --git a/mypy-baseline.txt b/mypy-baseline.txt
@@ -65,6 +65,7 @@ posthog/hogql/database/schema/persons.py:0: error: Argument "chain" to "Field" h
 posthog/hogql/database/schema/persons.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance
 posthog/hogql/database/schema/persons.py:0: note: Consider using "Sequence" instead, which is covariant
 posthog/hogql/parser.py:0: error: Key expression in dictionary comprehension has incompatible type "str"; expected type "Literal['expr', 'order_expr', 'select']"  [misc]
+posthog/hogql/parser.py:0: error: Statement is unreachable  [unreachable]
 posthog/hogql/parser.py:0: error: Item "None" of "list[Expr] | None" has no attribute "__iter__" (not iterable)  [union-attr]
 posthog/hogql/parser.py:0: error: "None" has no attribute "text"  [attr-defined]
 posthog/hogql/parser.py:0: error: "None" has no attribute "text"  [attr-defined]

diff --git a/posthog/hogql/grammar/HogQLParser.g4 b/posthog/hogql/grammar/HogQLParser.g4
@@ -9,7 +9,8 @@ options {
 select: (selectUnionStmt | selectStmt | hogqlxTagElement) EOF;
 
 selectUnionStmt: selectStmtWithParens (UNION ALL selectStmtWithParens)*;
-selectStmtWithParens: selectStmt | LPAREN selectUnionStmt RPAREN;
+selectStmtWithParens: selectStmt | LPAREN selectUnionStmt RPAREN | placeholder;
+
 selectStmt:
     with=withClause?
     SELECT DISTINCT? topClause?

diff --git a/posthog/hogql/grammar/HogQLParser.interp b/posthog/hogql/grammar/HogQLParser.interp
diff --git a/posthog/hogql/grammar/HogQLParser.py b/posthog/hogql/grammar/HogQLParser.py
diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py
@@ -172,14 +172,16 @@ def visitSelectUnionStmt(self, ctx: HogQLParser.SelectUnionStmtContext):
                 flattened_queries.append(query)
             elif isinstance(query, ast.SelectUnionQuery):
                 flattened_queries.extend(query.select_queries)
+            elif isinstance(query, ast.Placeholder):
+                flattened_queries.append(query)
             else:
                 raise Exception(f"Unexpected query node type {type(query).__name__}")
         if len(flattened_queries) == 1:
             return flattened_queries[0]
         return ast.SelectUnionQuery(select_queries=flattened_queries)
 
     def visitSelectStmtWithParens(self, ctx: HogQLParser.SelectStmtWithParensContext):
-        return self.visit(ctx.selectStmt() or ctx.selectUnionStmt())
+        return self.visit(ctx.selectStmt() or ctx.selectUnionStmt() or ctx.placeholder())
 
     def visitSelectStmt(self, ctx: HogQLParser.SelectStmtContext):
         select_query = ast.SelectQuery(

diff --git a/posthog/hogql/test/_test_parser.py b/posthog/hogql/test/_test_parser.py
@@ -1467,7 +1467,7 @@ def test_malformed_sql(self):
             query = "SELEC 2"
             with self.assertRaisesMessage(
                 SyntaxException,
-                "mismatched input 'SELEC' expecting {SELECT, WITH, '(', '<'}",
+                "mismatched input 'SELEC' expecting {SELECT, WITH, '{', '(', '<'}",
             ) as e:
                 self._select(query)
             self.assertEqual(e.exception.start, 0)

diff --git a/posthog/hogql/test/test_metadata.py b/posthog/hogql/test/test_metadata.py
@@ -79,7 +79,7 @@ def test_metadata_valid_expr_select(self):
                 "inputSelect": "timestamp",
                 "errors": [
                     {
-                        "message": "mismatched input 'timestamp' expecting {SELECT, WITH, '(', '<'}",
+                        "message": "mismatched input 'timestamp' expecting {SELECT, WITH, '{', '(', '<'}",
                         "start": 0,
                         "end": 9,
                         "fix": None,

diff --git a/posthog/hogql_queries/insights/stickiness_query_runner.py b/posthog/hogql_queries/insights/stickiness_query_runner.py
@@ -125,7 +125,7 @@ def to_query(self) -> List[ast.SelectQuery]:  # type: ignore
                             SELECT 0 as aggregation_target, (number + 1) as num_intervals
                             FROM numbers(dateDiff({interval}, {date_from} - {interval_subtract}, {date_to}))
                             UNION ALL
-                            SELECT * FROM ({events_query})
+                            {events_query}
                         )
                         GROUP BY num_intervals
                         ORDER BY num_intervals

diff --git a/requirements.in b/requirements.in
@@ -96,5 +96,5 @@ django-two-factor-auth==1.14.0
 phonenumberslite==8.13.6
 openai==0.27.8
 nh3==0.2.14
-hogql-parser==1.0.2
+hogql-parser==1.0.3
 urllib3[secure,socks]==1.26.18
diff --git a/requirements.txt b/requirements.txt
@@ -280,7 +280,7 @@ h11==0.13.0
     # via wsproto
 hexbytes==1.0.0
     # via dlt
-hogql-parser==1.0.2
+hogql-parser==1.0.3
     # via -r requirements.in
 humanize==4.9.0
     # via dlt