feat: add union distinct and intersect distinct to hogql parser

hogql_parser/HogQLParser.h

@@ -591,6 +591,7 @@ class  HogQLParser : public antlr4::Parser {
     antlr4::tree::TerminalNode *EXCEPT();
     antlr4::tree::TerminalNode *UNION();
     antlr4::tree::TerminalNode *ALL();
+    antlr4::tree::TerminalNode *DISTINCT();
     antlr4::tree::TerminalNode *INTERSECT();
 
 

hogql_parser/parser.cpp

@@ -867,12 +867,16 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor {
         char* set_operator;
         if (subsequent->UNION() && subsequent->ALL()) {
             set_operator = "UNION ALL";
+        } else if (subsequent->UNION() && subsequent->DISTINCT()) {
+            set_operator = "UNION DISTINCT";
+        } else if (subsequent->INTERSECT() && subsequent->DISTINCT()) {
+            set_operator = "INTERSECT DISTINCT";
         } else if (subsequent->INTERSECT()) {
             set_operator = "INTERSECT";
         } else if (subsequent->EXCEPT()) {
             set_operator = "EXCEPT";
         } else {
-            throw SyntaxError("Set operator must be one of UNION ALL, INTERSECT, and EXCEPT");
+            throw SyntaxError("Set operator must be one of UNION ALL, UNION DISTINCT, INTERSECT, INTERSECT DISTINCT, and EXCEPT");
         }
         select_query = visitAsPyObject(subsequent->selectStmtWithParens());
         PyObject* query = build_ast_node("SelectSetNode", "{s:N,s:N}", "select_query", select_query, "set_operator", PyUnicode_FromString(set_operator));

hogql_parser/setup.py

@@ -32,7 +32,7 @@
 
 setup(
     name="hogql_parser",
-    version="1.0.46",
+    version="1.0.47",
     url="https://github.com/PostHog/posthog/tree/master/hogql_parser",
     author="PostHog Inc.",
     author_email="[email protected]",

posthog/hogql/ast.py

@@ -821,7 +821,7 @@ class SelectQuery(Expr):
     view_name: Optional[str] = None
 
 
-SetOperator = Literal["UNION ALL", "INTERSECT", "EXCEPT"]
+SetOperator = Literal["UNION ALL", "UNION DISTINCT", "INTERSECT", "INTERSECT DISTINCT", "EXCEPT"]
 
 
 @dataclass(kw_only=True)

posthog/hogql/grammar/HogQLParser.g4

@@ -55,7 +55,7 @@ select: (selectSetStmt | selectStmt | hogqlxTagElement) EOF;
 
 selectStmtWithParens: selectStmt | LPAREN selectSetStmt RPAREN | placeholder;
 
-subsequentSelectSetClause: (EXCEPT | UNION ALL | INTERSECT) selectStmtWithParens;
+subsequentSelectSetClause: (EXCEPT | UNION ALL | UNION DISTINCT | INTERSECT | INTERSECT DISTINCT) selectStmtWithParens;
 selectSetStmt: selectStmtWithParens (subsequentSelectSetClause)*;
 
 selectStmt:

posthog/hogql/parser.py

@@ -338,12 +338,18 @@ def visitSelectSetStmt(self, ctx: HogQLParser.SelectSetStmtContext):
         for subsequent in ctx.subsequentSelectSetClause():
             if subsequent.UNION() and subsequent.ALL():
                 union_type = "UNION ALL"
+            elif subsequent.UNION() and subsequent.DISTINCT():
+                union_type = "UNION DISTINCT"
+            elif subsequent.INTERSECT() and subsequent.DISTINCT():
+                union_type = "INTERSECT DISTINCT"
             elif subsequent.INTERSECT():
                 union_type = "INTERSECT"
             elif subsequent.EXCEPT():
                 union_type = "EXCEPT"
             else:
-                raise SyntaxError("Set operator must be one of UNION ALL, INTERSECT, and EXCEPT")
+                raise SyntaxError(
+                    "Set operator must be one of UNION ALL, UNION DISTINCT, INTERSECT, INTERSECT DISTINCT, and EXCEPT"
+                )
             select_query = self.visit(subsequent.selectStmtWithParens())
             select_queries.append(
                 SelectSetNode(select_query=select_query, set_operator=cast(ast.SetOperator, union_type))

posthog/hogql/test/test_printer.py

@@ -114,6 +114,14 @@ def test_to_printed_hogql(self):
             repsponse, f"SELECT\n    plus(1, 2),\n    3\nFROM\n    events\nLIMIT {MAX_SELECT_RETURNED_ROWS}"
         )
 
+    def test_union_distinct(self):
+        expr = parse_select("""select 1 as id union distinct select 2 as id""")
+        response = to_printed_hogql(expr, self.team)
+        self.assertEqual(
+            response,
+            f"SELECT\n    1 AS id\nLIMIT 50000\nUNION DISTINCT\nSELECT\n    2 AS id\nLIMIT {MAX_SELECT_RETURNED_ROWS}",
+        )
+
     def test_intersect(self):
         expr = parse_select("""select 1 as id intersect select 2 as id""")
         response = to_printed_hogql(expr, self.team)
@@ -122,6 +130,14 @@ def test_intersect(self):
             f"SELECT\n    1 AS id\nLIMIT 50000\nINTERSECT\nSELECT\n    2 AS id\nLIMIT {MAX_SELECT_RETURNED_ROWS}",
         )
 
+    def test_intersect_distinct(self):
+        expr = parse_select("""select 1 as id intersect distinct select 2 as id""")
+        response = to_printed_hogql(expr, self.team)
+        self.assertEqual(
+            response,
+            f"SELECT\n    1 AS id\nLIMIT 50000\nINTERSECT DISTINCT\nSELECT\n    2 AS id\nLIMIT {MAX_SELECT_RETURNED_ROWS}",
+        )
+
     def test_except(self):
         expr = parse_select("""select 1 as id except select 2 as id""")
         response = to_printed_hogql(expr, self.team)

requirements.in

@@ -106,7 +106,7 @@ phonenumberslite==8.13.6
 openai==1.51.2
 tiktoken==0.8.0
 nh3==0.2.14
-hogql-parser==1.0.46
+hogql-parser==1.0.47
 zxcvbn==4.4.28
 zstd==1.5.5.1
 xmlsec==1.3.13 # Do not change this version - it will break SAML

requirements.txt

@@ -283,7 +283,7 @@ h11==0.13.0
     #   wsproto
 hexbytes==1.0.0
     # via dlt
-hogql-parser==1.0.46
+hogql-parser==1.0.47
     # via -r requirements.in
 httpcore==1.0.2
     # via httpx