feat: allow placeholders in place of ratio expressions in hogql (#17973)

* Allow placeholders in place of ratio expressions in hogql * Added C++ changes for RatioExpr * Bump `hogql_parser` version * Upgrade ANTLR version to align with brew latest * Update query snapshots * Fix parser change message body The quotes weren't right, so the step failed instead of posting a comment. * Revert "Update query snapshots" This reverts commit 1302d3b. * Update MD5 sum * Bump version again * Use new hogql-parser version --------- Co-authored-by: Michael Matloka <[email protected]> Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Michael Matloka <[email protected]>
PostHog · Oct 19, 2023 · 8577e13 · 8577e13
1 parent c0df793
commit 8577e13
Show file tree

Hide file tree

Showing 24 changed files with 1,536 additions and 1,477 deletions.
diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml
@@ -82,9 +82,9 @@ runs:
           # changed (requirements.txt has the already-published version)
           run: |
               sudo apt-get install libboost-all-dev unzip cmake curl uuid pkg-config
-              curl https://www.antlr.org/download/antlr4-cpp-runtime-4.13.0-source.zip --output antlr4-source.zip
+              curl https://www.antlr.org/download/antlr4-cpp-runtime-4.13.1-source.zip --output antlr4-source.zip
               # Check that the downloaded archive is the expected runtime - a security measure
-              anltr_known_md5sum="ff214b65fb02e150b4f515d7983bca92"
+              anltr_known_md5sum="c875c148991aacd043f733827644a76f"
               antlr_found_ms5sum="$(md5sum antlr4-source.zip | cut -d' ' -f1)"
               if [[ "$anltr_known_md5sum" != "$antlr_found_ms5sum" ]]; then
                   echo "Unexpected MD5 sum of antlr4-source.zip!"

diff --git a/.github/workflows/build-hogql-parser.yml b/.github/workflows/build-hogql-parser.yml
@@ -44,8 +44,8 @@ jobs:
                   local=$(python hogql_parser/setup.py --version)
                   # TODO: Only comment if no comment alraedy exists for $local
                   if [[ "$published" == "$local" ]]; then
-                    MESSAGE_BODY="It looks like the code of `hogql-parser` has changed since last push, but its version stayed the same at $local. 👀\nMake sure to resolve this in `hogql_parser/setup.py` before merging!"
-                    curl -s -u posthog-bot:${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} -X POST -d "{ \"body\": \"$MESSAGE_BODY\" }" "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments"
+                    message_body="It looks like the code of \`hogql-parser\` has changed since last push, but its version stayed the same at $local. 👀\nMake sure to resolve this in \`hogql_parser/setup.py\` before merging!"
+                    curl -s -u posthog-bot:${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} -X POST -d "{ \"body\": \"$message_body\" }" "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments"
                   fi
 
     build-wheels:

diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml
@@ -159,7 +159,7 @@ jobs:
               env:
                   # Installing a version of ANTLR compatible with what's in Homebrew as of October 2023 (version 4.13),
                   # as apt-get is quite out of date. The same version must be set in hogql_parser/pyproject.toml
-                  ANTLR_VERSION: '4.13.0'
+                  ANTLR_VERSION: '4.13.1'
 
     check-migrations:
         needs: changes

diff --git a/hogql_parser/HogQLLexer.cpp b/hogql_parser/HogQLLexer.cpp
@@ -1,5 +1,5 @@
 
-// Generated from HogQLLexer.g4 by ANTLR 4.13.0
+// Generated from HogQLLexer.g4 by ANTLR 4.13.1
 
 
 #include "HogQLLexer.h"

diff --git a/hogql_parser/HogQLLexer.h b/hogql_parser/HogQLLexer.h
@@ -1,5 +1,5 @@
 
-// Generated from HogQLLexer.g4 by ANTLR 4.13.0
+// Generated from HogQLLexer.g4 by ANTLR 4.13.1
 
 #pragma once
 

diff --git a/hogql_parser/HogQLParser.cpp b/hogql_parser/HogQLParser.cpp
diff --git a/hogql_parser/HogQLParser.h b/hogql_parser/HogQLParser.h
@@ -1,5 +1,5 @@
 
-// Generated from HogQLParser.g4 by ANTLR 4.13.0
+// Generated from HogQLParser.g4 by ANTLR 4.13.1
 
 #pragma once
 
@@ -675,6 +675,7 @@ class  HogQLParser : public antlr4::Parser {
   public:
     RatioExprContext(antlr4::ParserRuleContext *parent, size_t invokingState);
     virtual size_t getRuleIndex() const override;
+    antlr4::tree::TerminalNode *PLACEHOLDER();
     std::vector<NumberLiteralContext *> numberLiteral();
     NumberLiteralContext* numberLiteral(size_t i);
     antlr4::tree::TerminalNode *SLASH();

diff --git a/hogql_parser/HogQLParser.interp b/hogql_parser/HogQLParser.interp
diff --git a/hogql_parser/HogQLParserBaseVisitor.cpp b/hogql_parser/HogQLParserBaseVisitor.cpp
@@ -1,5 +1,5 @@
 
-// Generated from HogQLParser.g4 by ANTLR 4.13.0
+// Generated from HogQLParser.g4 by ANTLR 4.13.1
 
 
 #include "HogQLParserBaseVisitor.h"

diff --git a/hogql_parser/HogQLParserBaseVisitor.h b/hogql_parser/HogQLParserBaseVisitor.h
@@ -1,5 +1,5 @@
 
-// Generated from HogQLParser.g4 by ANTLR 4.13.0
+// Generated from HogQLParser.g4 by ANTLR 4.13.1
 
 #pragma once
 

diff --git a/hogql_parser/HogQLParserVisitor.cpp b/hogql_parser/HogQLParserVisitor.cpp
@@ -1,5 +1,5 @@
 
-// Generated from HogQLParser.g4 by ANTLR 4.13.0
+// Generated from HogQLParser.g4 by ANTLR 4.13.1
 
 
 #include "HogQLParserVisitor.h"

diff --git a/hogql_parser/HogQLParserVisitor.h b/hogql_parser/HogQLParserVisitor.h
@@ -1,5 +1,5 @@
 
-// Generated from HogQLParser.g4 by ANTLR 4.13.0
+// Generated from HogQLParser.g4 by ANTLR 4.13.1
 
 #pragma once
 

diff --git a/hogql_parser/parser.cpp b/hogql_parser/parser.cpp
@@ -570,6 +570,12 @@ class HogQLParseTreeConverter : public HogQLParserBaseVisitor {
   }
 
   VISIT(RatioExpr) {
+    auto placeholder_ctx = ctx->PLACEHOLDER();
+    if (placeholder_ctx) {
+      string placeholder = unquote_string_terminal(placeholder_ctx);
+      return build_ast_node("Placeholder", "{s:s#}", "field", placeholder.data(), placeholder.size());
+    }
+
     auto number_literal_ctxs = ctx->numberLiteral();
 
     if (number_literal_ctxs.size() > 2) {

diff --git a/hogql_parser/pyproject.toml b/hogql_parser/pyproject.toml
@@ -26,9 +26,9 @@ before-build = [ # We need to install the libraries for each architecture separa
 before-all = [
     # manylinux_2_28 is based on AlmaLinux 8, which uses Fedora's dnf as its package manager
     "dnf install -y boost-devel unzip cmake curl uuid pkg-config",
-    "curl https://www.antlr.org/download/antlr4-cpp-runtime-4.13.0-source.zip --output antlr4-source.zip",
+    "curl https://www.antlr.org/download/antlr4-cpp-runtime-4.13.1-source.zip --output antlr4-source.zip",
     # Check that the downloaded archive is the expected runtime - a security measure
-    "anltr_known_md5sum=\"ff214b65fb02e150b4f515d7983bca92\"",
+    "anltr_known_md5sum=\"c875c148991aacd043f733827644a76f\"",
     "antlr_found_ms5sum=\"$(md5sum antlr4-source.zip | cut -d' ' -f1)\"",
     'if [[ "$anltr_known_md5sum" != "$antlr_found_ms5sum" ]]; then exit 64; fi',
     "unzip antlr4-source.zip -d antlr4-source && cd antlr4-source",

diff --git a/hogql_parser/setup.py b/hogql_parser/setup.py
@@ -32,7 +32,7 @@
 
 setup(
     name="hogql_parser",
-    version="0.1.8",
+    version="0.1.9",
     url="https://github.com/PostHog/posthog/tree/master/hogql_parser",
     author="PostHog Inc.",
     author_email="[email protected]",

diff --git a/posthog/hogql/grammar/HogQLLexer.py b/posthog/hogql/grammar/HogQLLexer.py
@@ -1,4 +1,4 @@
-# Generated from HogQLLexer.g4 by ANTLR 4.13.0
+# Generated from HogQLLexer.g4 by ANTLR 4.13.1
 from antlr4 import *
 from io import StringIO
 import sys
@@ -1252,7 +1252,7 @@ class HogQLLexer(Lexer):
 
     def __init__(self, input=None, output:TextIO = sys.stdout):
         super().__init__(input, output)
-        self.checkVersion("4.13.0")
+        self.checkVersion("4.13.1")
         self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache())
         self._actions = None
         self._predicates = None

diff --git a/posthog/hogql/grammar/HogQLParser.g4 b/posthog/hogql/grammar/HogQLParser.g4
@@ -70,7 +70,7 @@ joinConstraintClause
 sampleClause: SAMPLE ratioExpr (OFFSET ratioExpr)?;
 orderExprList: orderExpr (COMMA orderExpr)*;
 orderExpr: columnExpr (ASCENDING | DESCENDING | DESC)? (NULLS (FIRST | LAST))? (COLLATE STRING_LITERAL)?;
-ratioExpr: numberLiteral (SLASH numberLiteral)?;
+ratioExpr: PLACEHOLDER | numberLiteral (SLASH numberLiteral)?;
 settingExprList: settingExpr (COMMA settingExpr)*;
 settingExpr: identifier EQ_SINGLE literal;
 

diff --git a/posthog/hogql/grammar/HogQLParser.interp b/posthog/hogql/grammar/HogQLParser.interp