Merge branch 'master' into hogql-dateadd

.devcontainer/Dockerfile

@@ -1,3 +1,7 @@
+# hadolint global ignore=DL3004
+
+# hadolint doesn't like changes to this file, but it is only used for local dev
+
 # Defines the environment you're dropped into with codespaces
 # I've take
 # https://github.com/microsoft/vscode-dev-containers/blob/main/containers/python-3/.devcontainer/Dockerfile
@@ -7,7 +11,7 @@
 # experience as rich as possible. Perhaps later down the line it might be worth
 # rolling our own
 #
-FROM mcr.microsoft.com/vscode/devcontainers/python:3.10-bullseye
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.11-bullseye
 
 # Make sure all exit codes on pipes cause failures
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]

.github/actions/run-backend-tests/action.yml

@@ -6,7 +6,7 @@ name: Run Django tests
 inputs:
     python-version:
         required: true
-        description: Python version, e.g. 3.10.10
+        description: Python version, e.g. 3.11.9
     clickhouse-server-image:
         required: true
         description: ClickHouse server image tag, e.g. clickhouse/clickhouse-server:latest

.github/workflows/benchmark.yml

@@ -54,7 +54,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/build-hogql-parser.yml

@@ -80,6 +80,21 @@ jobs:
               with:
                   python-version: '3.11'
 
+            # # This is an alternative way to install Python 3.11 on ARM if the above fails
+            # - if: ${{ endsWith(matrix.os, '-arm') }}
+            #   name: Install Python 3.11 on ARM (compile from source)
+            #   run: |
+            #       sudo apt-get update
+            #       sudo apt-get install -y build-essential libssl-dev zlib1g-dev \
+            #       libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev \
+            #       libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev
+            #       wget https://www.python.org/ftp/python/3.11.0/Python-3.11.0.tar.xz
+            #       tar -xf Python-3.11.0.tar.xz
+            #       cd Python-3.11.0
+            #       ./configure --enable-optimizations
+            #       make -j 2
+            #       sudo make altinstall
+
             - name: Build sdist
               if: matrix.os == 'ubuntu-22.04' # Only build the sdist once
               run: cd hogql_parser && python setup.py sdist

.github/workflows/ci-backend-update-test-timing.yml

@@ -28,7 +28,7 @@ jobs:
                   concurrency: 1
                   group: 1
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
-                  python-version: '3.10.10'
+                  python-version: '3.11.9'
                   clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
                   segment: 'FOSS'
                   person-on-events: false

.github/workflows/ci-backend.yml

@@ -108,7 +108,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -163,7 +163,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -177,22 +177,21 @@ jobs:
                   sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
 
             # First running migrations from master, to simulate the real-world scenario
-
-            - name: Checkout master
-              uses: actions/checkout@v3
-              with:
-                  ref: master
-
-            - name: Install python dependencies for master
-              run: |
-                  uv pip install --system -r requirements.txt -r requirements-dev.txt
-
-            - name: Run migrations up to master
-              run: |
-                  python manage.py migrate
+            # Commented out to move to Python 3.11. Uncomment after deploy.
+            #            - name: Checkout master
+            #              uses: actions/checkout@v3
+            #              with:
+            #                  ref: master
+            #
+            #            - name: Install python dependencies for master
+            #              run: |
+            #                  uv pip install --system -r requirements.txt -r requirements-dev.txt
+            #
+            #            - name: Run migrations up to master
+            #              run: |
+            #                  python manage.py migrate
 
             # Now we can consider this PR's migrations
-
             - name: Checkout this PR
               uses: actions/checkout@v3
 
@@ -204,22 +203,24 @@ jobs:
               run: |
                   python manage.py migrate
 
-            - name: Check migrations
-              run: |
-                  python manage.py makemigrations --check --dry-run
-                  git fetch origin master
-                  # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified
-                  # With `--name-status` we get the same, but including a column for status, respectively: A, D, M
-                  # In this check we exclusively care about files that were
-                  # added (A) in posthog/migrations/. We also want to ignore
-                  # initial migrations (0001_*) as these are guaranteed to be
-                  # run on initial setup where there is no data.
-                  git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe
-
-            - name: Check CH migrations
-              run: |
-                  # Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/
-                  git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | awk '{print $2}' |  python manage.py test_ch_migrations_are_safe
+    # Commented out to move to Python 3.11. Uncomment after deploy.
+    #
+    #            - name: Check migrations
+    #              run: |
+    #                  python manage.py makemigrations --check --dry-run
+    #                  git fetch origin master
+    #                  # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified
+    #                  # With `--name-status` we get the same, but including a column for status, respectively: A, D, M
+    #                  # In this check we exclusively care about files that were
+    #                  # added (A) in posthog/migrations/. We also want to ignore
+    #                  # initial migrations (0001_*) as these are guaranteed to be
+    #                  # run on initial setup where there is no data.
+    #                  git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe
+    #
+    #            - name: Check CH migrations
+    #              run: |
+    #                  # Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/
+    #                  git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | awk '{print $2}' |  python manage.py test_ch_migrations_are_safe
 
     django:
         needs: changes
@@ -231,7 +232,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                python-version: ['3.10.10']
+                python-version: ['3.11.9']
                 clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.5.81-alpine']
                 segment: ['Core']
                 person-on-events: [false, true]
@@ -242,7 +243,7 @@ jobs:
                     - segment: 'Temporal'
                       person-on-events: false
                       clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
-                      python-version: '3.10.10'
+                      python-version: '3.11.9'
                       concurrency: 1
                       group: 1
 
@@ -330,7 +331,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-hog.yml

@@ -70,7 +70,7 @@ jobs:
               if: needs.changes.outputs.hog == 'true'
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-plugin-server.yml

@@ -115,7 +115,7 @@ jobs:
               if: needs.changes.outputs.plugin-server == 'true'
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -207,7 +207,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.run/Celery Beat.run.xml

@@ -14,6 +14,7 @@
       <env name="REPLAY_EMBEDDINGS_ALLOWED_TEAM" value="1,2,3" />
     </envs>
     <option name="SDK_HOME" value="" />
+    <option name="SDK_NAME" value="Python 3.11 (posthog)" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/env/bin" />
     <option name="IS_MODULE_SDK" value="true" />
     <option name="ADD_CONTENT_ROOTS" value="true" />

.run/Celery Threads.run.xml

@@ -14,6 +14,7 @@
       <env name="REPLAY_EMBEDDINGS_ALLOWED_TEAM" value="1,2,3" />
     </envs>
     <option name="SDK_HOME" value="$PROJECT_DIR$/env/bin/python" />
+    <option name="SDK_NAME" value="Python 3.11 (posthog)" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/env/bin" />
     <option name="IS_MODULE_SDK" value="true" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
@@ -28,4 +29,4 @@
     <option name="INPUT_FILE" value="" />
     <method v="2" />
   </configuration>
-</component>
+</component>

.run/Celery.run.xml

@@ -5,17 +5,18 @@
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
     <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
       <env name="CLICKHOUSE_SECURE" value="False" />
       <env name="DATABASE_URL" value="postgres://posthog:posthog@localhost:5432/posthog" />
       <env name="DEBUG" value="1" />
       <env name="KAFKA_HOSTS" value="localhost" />
-      <env name="PYTHONUNBUFFERED" value="1" />
       <env name="SKIP_SERVICE_VERSION_REQUIREMENTS" value="1" />
       <env name="REPLAY_EMBEDDINGS_ALLOWED_TEAM" value="1,2,3" />
     </envs>
     <option name="SDK_HOME" value="$PROJECT_DIR$/env/bin/python" />
+    <option name="SDK_NAME" value="Python 3.11 (posthog)" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/env/bin" />
-    <option name="IS_MODULE_SDK" value="false" />
+    <option name="IS_MODULE_SDK" value="true" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />

.run/PostHog.run.xml

@@ -1,9 +1,11 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="PostHog" type="Python.DjangoServer" factoryName="Django server">
     <module name="posthog" />
+    <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
     <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
       <env name="BILLING_SERVICE_URL" value="https://billing.dev.posthog.dev" />
       <env name="CAPTURE_TIME_TO_SEE_DATA" value="0" />
       <env name="CLICKHOUSE_SECURE" value="False" />
@@ -14,14 +16,13 @@
       <env name="KEA_VERBOSE_LOGGING" value="false" />
       <env name="PRINT_SQL" value="1" />
       <env name="PYDEVD_USE_CYTHON" value="NO" />
-      <env name="PYTHONUNBUFFERED" value="1" />
       <env name="SESSION_RECORDING_KAFKA_COMPRESSION" value="gzip" />
       <env name="SESSION_RECORDING_KAFKA_HOSTS" value="localhost" />
       <env name="SESSION_RECORDING_KAFKA_MAX_REQUEST_SIZE_BYTES" value="20971520" />
       <env name="SKIP_SERVICE_VERSION_REQUIREMENTS" value="1" />
     </envs>
     <option name="SDK_HOME" value="$PROJECT_DIR$/env/bin/python" />
-    <option name="SDK_NAME" value="Python 3.10 (posthog)" />
+    <option name="SDK_NAME" value="Python 3.11 (posthog)" />
     <option name="WORKING_DIRECTORY" value="" />
     <option name="IS_MODULE_SDK" value="false" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
@@ -47,4 +48,4 @@
     <option name="customRunCommand" value="" />
     <method v="2" />
   </configuration>
-</component>
+</component>

bin/build-schema-python.sh

@@ -4,7 +4,7 @@ set -e
 
 # Generate schema.py from schema.json
 datamodel-codegen \
-    --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.10 --disable-timestamp \
+    --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.11 --disable-timestamp \
     --use-one-literal-as-default --use-default --use-default-kwarg --use-subclass-enum \
     --input frontend/src/queries/schema.json --input-file-type jsonschema \
     --output posthog/schema.py --output-model-type pydantic_v2.BaseModel \
@@ -29,3 +29,9 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
 else
     sed -i -e 's/Optional\[PropertyOperator\] = \("[A-Za-z_]*"\)/Optional[PropertyOperator] = PropertyOperator(\1)/g' posthog/schema.py
 fi
+
+# Replace class Foo(str, Enum) with class Foo(StrEnum) for proper handling in format strings in python 3.11
+# Remove this when https://github.com/koxudaxi/datamodel-code-generator/issues/1313 is resolved
+
+sed -i -e 's/str, Enum/StrEnum/g' posthog/schema.py
+sed -i 's/from enum import Enum/from enum import Enum, StrEnum/g' posthog/schema.py

bin/deploy-hobby

@@ -184,7 +184,7 @@ if ! command -v docker &> /dev/null; then
     # Setup Docker
     sudo apt install -y apt-transport-https ca-certificates curl software-properties-common
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -E apt-key add -
-    sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
+    sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable"
     sudo apt update
     sudo apt-cache policy docker-ce
     sudo apt install -y docker-ce

ee/api/test/__snapshots__/test_time_to_see_data.ambr

@@ -20,7 +20,7 @@
               "first_name": "",
               "last_name": "",
               "email": "",
-              "is_email_verified": false
+              "is_email_verified": null
           }
       },
       "children": [

hogql_parser/HogQLParser.cpp

@@ -250,8 +250,8 @@ void hogqlparserParserInitialize() {
   	1,0,0,0,194,192,1,0,0,0,194,195,1,0,0,0,195,198,1,0,0,0,196,194,1,0,0,
   	0,197,199,5,112,0,0,198,197,1,0,0,0,198,199,1,0,0,0,199,9,1,0,0,0,200,
   	210,3,12,6,0,201,210,3,14,7,0,202,210,3,16,8,0,203,210,3,18,9,0,204,210,
-  	3,20,10,0,205,210,3,22,11,0,206,210,3,24,12,0,207,210,3,26,13,0,208,210,
-  	3,28,14,0,209,200,1,0,0,0,209,201,1,0,0,0,209,202,1,0,0,0,209,203,1,0,
+  	3,20,10,0,205,210,3,22,11,0,206,210,3,28,14,0,207,210,3,24,12,0,208,210,
+  	3,26,13,0,209,200,1,0,0,0,209,201,1,0,0,0,209,202,1,0,0,0,209,203,1,0,
   	0,0,209,204,1,0,0,0,209,205,1,0,0,0,209,206,1,0,0,0,209,207,1,0,0,0,209,
   	208,1,0,0,0,210,11,1,0,0,0,211,213,5,70,0,0,212,214,3,4,2,0,213,212,1,
   	0,0,0,213,214,1,0,0,0,214,216,1,0,0,0,215,217,5,145,0,0,216,215,1,0,0,
@@ -1118,6 +1118,10 @@ HogQLParser::VarAssignmentContext* HogQLParser::StatementContext::varAssignment(
   return getRuleContext<HogQLParser::VarAssignmentContext>(0);
 }
 
+HogQLParser::BlockContext* HogQLParser::StatementContext::block() {
+  return getRuleContext<HogQLParser::BlockContext>(0);
+}
+
 HogQLParser::ExprStmtContext* HogQLParser::StatementContext::exprStmt() {
   return getRuleContext<HogQLParser::ExprStmtContext>(0);
 }
@@ -1126,10 +1130,6 @@ HogQLParser::EmptyStmtContext* HogQLParser::StatementContext::emptyStmt() {
   return getRuleContext<HogQLParser::EmptyStmtContext>(0);
 }
 
-HogQLParser::BlockContext* HogQLParser::StatementContext::block() {
-  return getRuleContext<HogQLParser::BlockContext>(0);
-}
-
 
 size_t HogQLParser::StatementContext::getRuleIndex() const {
   return HogQLParser::RuleStatement;
@@ -1203,21 +1203,21 @@ HogQLParser::StatementContext* HogQLParser::statement() {
     case 7: {
       enterOuterAlt(_localctx, 7);
       setState(206);
-      exprStmt();
+      block();
       break;
     }
 
     case 8: {
       enterOuterAlt(_localctx, 8);
       setState(207);
-      emptyStmt();
+      exprStmt();
       break;
     }
 
     case 9: {
       enterOuterAlt(_localctx, 9);
       setState(208);
-      block();
+      emptyStmt();
       break;
     }
 

hogql_parser/HogQLParser.h

@@ -258,9 +258,9 @@ class  HogQLParser : public antlr4::Parser {
     ForStmtContext *forStmt();
     FuncStmtContext *funcStmt();
     VarAssignmentContext *varAssignment();
+    BlockContext *block();
     ExprStmtContext *exprStmt();
     EmptyStmtContext *emptyStmt();
-    BlockContext *block();
 
 
     virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;