chore: upgrade python to 3.11 🐍 (#22932)

🐍

.devcontainer/Dockerfile

@@ -1,3 +1,7 @@
+# hadolint global ignore=DL3004
+
+# hadolint doesn't like changes to this file, but it is only used for local dev
+
 # Defines the environment you're dropped into with codespaces
 # I've take
 # https://github.com/microsoft/vscode-dev-containers/blob/main/containers/python-3/.devcontainer/Dockerfile
@@ -7,7 +11,7 @@
 # experience as rich as possible. Perhaps later down the line it might be worth
 # rolling our own
 #
-FROM mcr.microsoft.com/vscode/devcontainers/python:3.10-bullseye
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.11-bullseye
 
 # Make sure all exit codes on pipes cause failures
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]

.github/actions/run-backend-tests/action.yml

@@ -6,7 +6,7 @@ name: Run Django tests
 inputs:
     python-version:
         required: true
-        description: Python version, e.g. 3.10.10
+        description: Python version, e.g. 3.11.9
     clickhouse-server-image:
         required: true
         description: ClickHouse server image tag, e.g. clickhouse/clickhouse-server:latest

.github/workflows/benchmark.yml

@@ -54,7 +54,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-backend-update-test-timing.yml

@@ -28,7 +28,7 @@ jobs:
                   concurrency: 1
                   group: 1
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
-                  python-version: '3.10.10'
+                  python-version: '3.11.9'
                   clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
                   segment: 'FOSS'
                   person-on-events: false

.github/workflows/ci-backend.yml

@@ -108,7 +108,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -163,7 +163,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -177,22 +177,21 @@ jobs:
                   sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
 
             # First running migrations from master, to simulate the real-world scenario
-
-            - name: Checkout master
-              uses: actions/checkout@v3
-              with:
-                  ref: master
-
-            - name: Install python dependencies for master
-              run: |
-                  uv pip install --system -r requirements.txt -r requirements-dev.txt
-
-            - name: Run migrations up to master
-              run: |
-                  python manage.py migrate
+            # Commented out to move to Python 3.11. Uncomment after deploy.
+            #            - name: Checkout master
+            #              uses: actions/checkout@v3
+            #              with:
+            #                  ref: master
+            #
+            #            - name: Install python dependencies for master
+            #              run: |
+            #                  uv pip install --system -r requirements.txt -r requirements-dev.txt
+            #
+            #            - name: Run migrations up to master
+            #              run: |
+            #                  python manage.py migrate
 
             # Now we can consider this PR's migrations
-
             - name: Checkout this PR
               uses: actions/checkout@v3
 
@@ -204,22 +203,24 @@ jobs:
               run: |
                   python manage.py migrate
 
-            - name: Check migrations
-              run: |
-                  python manage.py makemigrations --check --dry-run
-                  git fetch origin master
-                  # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified
-                  # With `--name-status` we get the same, but including a column for status, respectively: A, D, M
-                  # In this check we exclusively care about files that were
-                  # added (A) in posthog/migrations/. We also want to ignore
-                  # initial migrations (0001_*) as these are guaranteed to be
-                  # run on initial setup where there is no data.
-                  git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe
-
-            - name: Check CH migrations
-              run: |
-                  # Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/
-                  git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | awk '{print $2}' |  python manage.py test_ch_migrations_are_safe
+    # Commented out to move to Python 3.11. Uncomment after deploy.
+    #
+    #            - name: Check migrations
+    #              run: |
+    #                  python manage.py makemigrations --check --dry-run
+    #                  git fetch origin master
+    #                  # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified
+    #                  # With `--name-status` we get the same, but including a column for status, respectively: A, D, M
+    #                  # In this check we exclusively care about files that were
+    #                  # added (A) in posthog/migrations/. We also want to ignore
+    #                  # initial migrations (0001_*) as these are guaranteed to be
+    #                  # run on initial setup where there is no data.
+    #                  git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe
+    #
+    #            - name: Check CH migrations
+    #              run: |
+    #                  # Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/
+    #                  git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | awk '{print $2}' |  python manage.py test_ch_migrations_are_safe
 
     django:
         needs: changes
@@ -231,7 +232,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                python-version: ['3.10.10']
+                python-version: ['3.11.9']
                 clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.5.81-alpine']
                 segment: ['Core']
                 person-on-events: [false, true]
@@ -242,7 +243,7 @@ jobs:
                     - segment: 'Temporal'
                       person-on-events: false
                       clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
-                      python-version: '3.10.10'
+                      python-version: '3.11.9'
                       concurrency: 1
                       group: 1
 
@@ -330,7 +331,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-hog.yml

@@ -70,7 +70,7 @@ jobs:
               if: needs.changes.outputs.hog == 'true'
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-plugin-server.yml

@@ -115,7 +115,7 @@ jobs:
               if: needs.changes.outputs.plugin-server == 'true'
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
@@ -207,7 +207,7 @@ jobs:
             - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.10.10
+                  python-version: 3.11.9
                   cache: 'pip'
                   cache-dependency-path: '**/requirements*.txt'
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.run/Celery Beat.run.xml

@@ -14,6 +14,7 @@
       <env name="REPLAY_EMBEDDINGS_ALLOWED_TEAM" value="1,2,3" />
     </envs>
     <option name="SDK_HOME" value="" />
+    <option name="SDK_NAME" value="Python 3.11 (posthog)" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/env/bin" />
     <option name="IS_MODULE_SDK" value="true" />
     <option name="ADD_CONTENT_ROOTS" value="true" />

.run/Celery Threads.run.xml

@@ -14,6 +14,7 @@
       <env name="REPLAY_EMBEDDINGS_ALLOWED_TEAM" value="1,2,3" />
     </envs>
     <option name="SDK_HOME" value="$PROJECT_DIR$/env/bin/python" />
+    <option name="SDK_NAME" value="Python 3.11 (posthog)" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/env/bin" />
     <option name="IS_MODULE_SDK" value="true" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
@@ -28,4 +29,4 @@
     <option name="INPUT_FILE" value="" />
     <method v="2" />
   </configuration>
-</component>
+</component>

.run/Celery.run.xml

@@ -5,17 +5,18 @@
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
     <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
       <env name="CLICKHOUSE_SECURE" value="False" />
       <env name="DATABASE_URL" value="postgres://posthog:posthog@localhost:5432/posthog" />
       <env name="DEBUG" value="1" />
       <env name="KAFKA_HOSTS" value="localhost" />
-      <env name="PYTHONUNBUFFERED" value="1" />
       <env name="SKIP_SERVICE_VERSION_REQUIREMENTS" value="1" />
       <env name="REPLAY_EMBEDDINGS_ALLOWED_TEAM" value="1,2,3" />
     </envs>
     <option name="SDK_HOME" value="$PROJECT_DIR$/env/bin/python" />
+    <option name="SDK_NAME" value="Python 3.11 (posthog)" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/env/bin" />
-    <option name="IS_MODULE_SDK" value="false" />
+    <option name="IS_MODULE_SDK" value="true" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />

.run/PostHog.run.xml

@@ -1,9 +1,11 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="PostHog" type="Python.DjangoServer" factoryName="Django server">
     <module name="posthog" />
+    <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
     <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
       <env name="BILLING_SERVICE_URL" value="https://billing.dev.posthog.dev" />
       <env name="CAPTURE_TIME_TO_SEE_DATA" value="0" />
       <env name="CLICKHOUSE_SECURE" value="False" />
@@ -14,14 +16,13 @@
       <env name="KEA_VERBOSE_LOGGING" value="false" />
       <env name="PRINT_SQL" value="1" />
       <env name="PYDEVD_USE_CYTHON" value="NO" />
-      <env name="PYTHONUNBUFFERED" value="1" />
       <env name="SESSION_RECORDING_KAFKA_COMPRESSION" value="gzip" />
       <env name="SESSION_RECORDING_KAFKA_HOSTS" value="localhost" />
       <env name="SESSION_RECORDING_KAFKA_MAX_REQUEST_SIZE_BYTES" value="20971520" />
       <env name="SKIP_SERVICE_VERSION_REQUIREMENTS" value="1" />
     </envs>
     <option name="SDK_HOME" value="$PROJECT_DIR$/env/bin/python" />
-    <option name="SDK_NAME" value="Python 3.10 (posthog)" />
+    <option name="SDK_NAME" value="Python 3.11 (posthog)" />
     <option name="WORKING_DIRECTORY" value="" />
     <option name="IS_MODULE_SDK" value="false" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
@@ -47,4 +48,4 @@
     <option name="customRunCommand" value="" />
     <method v="2" />
   </configuration>
-</component>
+</component>

bin/build-schema-python.sh

@@ -4,7 +4,7 @@ set -e
 
 # Generate schema.py from schema.json
 datamodel-codegen \
-    --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.10 --disable-timestamp \
+    --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.11 --disable-timestamp \
     --use-one-literal-as-default --use-default --use-default-kwarg --use-subclass-enum \
     --input frontend/src/queries/schema.json --input-file-type jsonschema \
     --output posthog/schema.py --output-model-type pydantic_v2.BaseModel \
@@ -29,3 +29,9 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
 else
     sed -i -e 's/Optional\[PropertyOperator\] = \("[A-Za-z_]*"\)/Optional[PropertyOperator] = PropertyOperator(\1)/g' posthog/schema.py
 fi
+
+# Replace class Foo(str, Enum) with class Foo(StrEnum) for proper handling in format strings in python 3.11
+# Remove this when https://github.com/koxudaxi/datamodel-code-generator/issues/1313 is resolved
+
+sed -i -e 's/str, Enum/StrEnum/g' posthog/schema.py
+sed -i 's/from enum import Enum/from enum import Enum, StrEnum/g' posthog/schema.py

bin/deploy-hobby

@@ -184,7 +184,7 @@ if ! command -v docker &> /dev/null; then
     # Setup Docker
     sudo apt install -y apt-transport-https ca-certificates curl software-properties-common
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -E apt-key add -
-    sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
+    sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable"
     sudo apt update
     sudo apt-cache policy docker-ce
     sudo apt install -y docker-ce

ee/api/test/__snapshots__/test_time_to_see_data.ambr

@@ -20,7 +20,7 @@
               "first_name": "",
               "last_name": "",
               "email": "",
-              "is_email_verified": false
+              "is_email_verified": null
           }
       },
       "children": [

mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.10
+python_version = 3.11
 plugins =
     mypy_django_plugin.main,
     mypy_drf_plugin.main,

posthog/api/comments.py

@@ -11,11 +11,13 @@
 
 from posthog.api.routing import TeamAndOrgViewSetMixin
 from posthog.api.shared import UserBasicSerializer
+from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer
 from posthog.models.comment import Comment
 
 
 class CommentSerializer(serializers.ModelSerializer):
     created_by = UserBasicSerializer(read_only=True)
+    deleted = ClassicBehaviorBooleanFieldSerializer()
 
     class Meta:
         model = Comment

posthog/api/feature_flag.py

@@ -23,6 +23,7 @@
 from posthog.api.shared import UserBasicSerializer
 from posthog.api.tagged_item import TaggedItemSerializerMixin, TaggedItemViewSetMixin
 from posthog.api.dashboards.dashboard import Dashboard
+from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer
 from posthog.auth import PersonalAPIKeyAuthentication, TemporaryTokenAuthentication
 from posthog.constants import FlagRequestType
 from posthog.event_usage import report_user_action
@@ -89,6 +90,9 @@ class FeatureFlagSerializer(TaggedItemSerializerMixin, serializers.HyperlinkedMo
     is_simple_flag = serializers.SerializerMethodField()
     rollout_percentage = serializers.SerializerMethodField()
 
+    ensure_experience_continuity = ClassicBehaviorBooleanFieldSerializer()
+    has_enriched_analytics = ClassicBehaviorBooleanFieldSerializer()
+
     experiment_set: serializers.PrimaryKeyRelatedField = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     surveys: serializers.SerializerMethodField = serializers.SerializerMethodField()
     features: serializers.SerializerMethodField = serializers.SerializerMethodField()

posthog/api/plugin.py

@@ -22,6 +22,7 @@
 
 from posthog.api.routing import TeamAndOrgViewSetMixin
 from posthog.api.shared import FiltersSerializer
+from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer
 from posthog.models import Plugin, PluginAttachment, PluginConfig, User
 from posthog.models.activity_logging.activity_log import (
     ActivityPage,
@@ -586,6 +587,8 @@ class PluginConfigSerializer(serializers.ModelSerializer):
     delivery_rate_24h = serializers.SerializerMethodField()
     error = serializers.SerializerMethodField()
 
+    deleted = ClassicBehaviorBooleanFieldSerializer()
+
     class Meta:
         model = PluginConfig
         fields = [

posthog/api/routing.py

@@ -36,6 +36,32 @@
 class DefaultRouterPlusPlus(ExtendedDefaultRouter):
     """DefaultRouter with optional trailing slash and drf-extensions nesting."""
 
+    # This is an override because of changes in djangorestframework 3.15, which is required for python 3.11
+    # changes taken from and explained here: https://github.com/nautobot/nautobot/pull/5546/files#diff-81850a2ccad5814aab4f477d447f85cc0a82e9c10fd88fd72327cda51a750471R30
+    def _register(self, prefix, viewset, basename=None):
+        """
+        Override DRF's BaseRouter.register() to bypass an unnecessary restriction added in version 3.15.0.
+        (Reference: https://github.com/encode/django-rest-framework/pull/8438)
+        """
+        if basename is None:
+            basename = self.get_default_basename(viewset)
+
+        # DRF:
+        # if self.is_already_registered(basename):
+        #     msg = (f'Router with basename "{basename}" is already registered. '
+        #            f'Please provide a unique basename for viewset "{viewset}"')
+        #     raise ImproperlyConfigured(msg)
+        #
+        # We bypass this because we have at least one use case (/api/extras/jobs/) where we are *intentionally*
+        # registering two viewsets with the same basename, but have carefully defined them so as not to conflict.
+
+        # resuming standard DRF code...
+        self.registry.append((prefix, viewset, basename))
+
+        # invalidate the urls cache
+        if hasattr(self, "_urls"):
+            del self._urls
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.trailing_slash = r"/?"