Merge branch 'master' into by/refactor-billing-page-error-display

.github/workflows/ci-backend.yml

@@ -417,6 +417,7 @@ jobs:
                   echo running_time_run_id=${run_id} >> $GITHUB_ENV
                   echo running_time_run_started_at=${run_started_at} >> $GITHUB_ENV
             - name: Capture running time to PostHog
+              if: github.repository == 'PostHog/posthog'
               uses: PostHog/[email protected]
               with:
                   posthog-token: ${{secrets.POSTHOG_API_TOKEN}}

.github/workflows/ci-e2e.yml

@@ -305,6 +305,7 @@ jobs:
                   echo running_time_run_id=${run_id} >> $GITHUB_ENV
                   echo running_time_run_started_at=${run_started_at} >> $GITHUB_ENV
             - name: Capture running time to PostHog
+              if: github.repository == 'PostHog/posthog'
               uses: PostHog/[email protected]
               with:
                   posthog-token: ${{secrets.POSTHOG_API_TOKEN}}

.github/workflows/report-pr-age.yml

@@ -23,6 +23,7 @@ jobs:
                       echo is_revert=false >> $GITHUB_ENV
                   fi
             - name: Capture PR age to PostHog
+              if: github.repository == 'PostHog/posthog'
               uses: PostHog/[email protected]
               with:
                   posthog-token: ${{secrets.POSTHOG_API_TOKEN}}

bin/build-schema-python.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Generate schema.py from schema.json
+datamodel-codegen \
+    --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.10 --disable-timestamp \
+    --use-one-literal-as-default --use-default --use-default-kwarg --use-subclass-enum \
+    --input frontend/src/queries/schema.json --input-file-type jsonschema \
+    --output posthog/schema.py --output-model-type pydantic_v2.BaseModel
+
+# Format schema.py
+ruff format posthog/schema.py
+
+# Check schema.py and autofix
+ruff check --fix posthog/schema.py
+
+# HACK: Datamodel-codegen output for enum-type fields with a default is invalid – the default value is a plain string,
+# and not the expected enum member. We fix this using sed, which is pretty hacky, but does the job.
+# Specifically, we need to replace `Optional[PropertyOperator] = "exact"`
+# with `Optional[PropertyOperator] = PropertyOperator("exact")` to make the default value valid.
+# Remove this when https://github.com/koxudaxi/datamodel-code-generator/issues/1929 is resolved.
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    # sed needs `-i` to be followed by `''` on macOS
+    sed -i '' -e 's/Optional\[PropertyOperator\] = \("[A-Za-z_]*"\)/Optional[PropertyOperator] = PropertyOperator(\1)/g' posthog/schema.py
+else
+    sed -i -e 's/Optional\[PropertyOperator\] = \("[A-Za-z_]*"\)/Optional[PropertyOperator] = PropertyOperator(\1)/g' posthog/schema.py
+fi

bin/migrate_kafka_data.py

@@ -21,7 +21,6 @@
 
 import argparse
 import sys
-from typing import List
 
 from kafka import KafkaAdminClient, KafkaConsumer, KafkaProducer
 from kafka.errors import KafkaError
@@ -192,7 +191,7 @@ def handle(**options):
             print("Polling for messages")  # noqa: T201
             messages_by_topic = consumer.poll(timeout_ms=timeout_ms)
 
-            futures: List[FutureRecordMetadata] = []
+            futures: list[FutureRecordMetadata] = []
 
             if not messages_by_topic:
                 break

cypress/e2e/dashboard.cy.ts

@@ -100,6 +100,40 @@ describe('Dashboard', () => {
         cy.get('[data-attr^="breadcrumb-Dashboard:"]').should('have.text', TEST_DASHBOARD_NAME + 'UnnamedCancelSave')
     })
 
+    const assertVariablesConfigurationScreenIsShown = (): void => {
+        cy.get('[data-attr="new-dashboard-chooser"]').contains('Unique variable name').should('exist')
+    }
+
+    it('Allow reselecting a dashboard after pressing back', () => {
+        cy.intercept('GET', /\/api\/projects\/\d+\/dashboard_templates/, (req) => {
+            req.reply((response) => {
+                response.body.results[0].variables = [
+                    {
+                        id: 'id',
+                        name: 'Unique variable name',
+                        type: 'event',
+                        default: {},
+                        required: true,
+                        description: 'description',
+                    },
+                ]
+                return response
+            })
+        })
+
+        // Request templates again.
+        cy.clickNavMenu('dashboards')
+
+        cy.get('[data-attr="new-dashboard"]').click()
+        cy.get('[data-attr="create-dashboard-from-template"]').click()
+        assertVariablesConfigurationScreenIsShown()
+
+        cy.contains('.LemonButton', 'Back').click()
+
+        cy.get('[data-attr="create-dashboard-from-template"]').click()
+        assertVariablesConfigurationScreenIsShown()
+    })
+
     it('Click on a dashboard item dropdown and view graph', () => {
         cy.get('[data-attr=dashboard-name]').contains('Web Analytics').click()
         cy.get('.InsightCard [data-attr=more-button]').first().click()

docker-compose.hobby.yml

@@ -139,6 +139,10 @@ services:
             - 7233:7233
         volumes:
             - ./posthog/docker/temporal/dynamicconfig:/etc/temporal/config/dynamicconfig
+    elasticsearch:
+        extends:
+            file: docker-compose.base.yml
+            service: elasticsearch
     temporal-admin-tools:
         extends:
             file: docker-compose.base.yml

ee/api/authentication.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Union
+from typing import Any, Union
 
 from django.core.exceptions import ValidationError as DjangoValidationError
 from django.http.response import HttpResponse
@@ -91,8 +91,8 @@ def auth_url(self):
 
     def _get_attr(
         self,
-        response_attributes: Dict[str, Any],
-        attribute_names: List[str],
+        response_attributes: dict[str, Any],
+        attribute_names: list[str],
         optional: bool = False,
     ) -> str:
         """

ee/api/dashboard_collaborator.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, cast
+from typing import Any, cast
 
 from django.db import IntegrityError
 from rest_framework import exceptions, mixins, serializers, viewsets
@@ -45,7 +45,7 @@ class Meta:
         ]
         read_only_fields = ["id", "dashboard_id", "user", "user"]
 
-    def validate(self, attrs: Dict[str, Any]) -> Dict[str, Any]:
+    def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
         dashboard: Dashboard = self.context["dashboard"]
         dashboard_permissions = self.user_permissions.dashboard(dashboard)
         if dashboard_permissions.effective_restriction_level <= Dashboard.RestrictionLevel.EVERYONE_IN_PROJECT_CAN_EDIT:
@@ -96,7 +96,7 @@ class DashboardCollaboratorViewSet(
     serializer_class = DashboardCollaboratorSerializer
     filter_rewrite_rules = {"team_id": "dashboard__team_id"}
 
-    def get_serializer_context(self) -> Dict[str, Any]:
+    def get_serializer_context(self) -> dict[str, Any]:
         context = super().get_serializer_context()
         try:
             context["dashboard"] = Dashboard.objects.get(id=context["dashboard_id"])

ee/api/role.py

@@ -1,4 +1,4 @@
-from typing import List, cast
+from typing import cast
 
 from django.db import IntegrityError
 from rest_framework import mixins, serializers, viewsets
@@ -76,7 +76,7 @@ def get_members(self, role: Role):
         return RoleMembershipSerializer(members, many=True).data
 
     def get_associated_flags(self, role: Role):
-        associated_flags: List[dict] = []
+        associated_flags: list[dict] = []
 
         role_access_objects = FeatureFlagRoleAccess.objects.filter(role=role).values_list("feature_flag_id")
         flags = FeatureFlag.objects.filter(id__in=role_access_objects)

ee/api/sentry_stats.py

@@ -1,5 +1,5 @@
 from datetime import datetime, timedelta
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Optional, Union
 
 import requests
 from django.http import HttpRequest, JsonResponse
@@ -9,8 +9,8 @@
 from posthog.models.instance_setting import get_instance_settings
 
 
-def get_sentry_stats(start_time: str, end_time: str) -> Tuple[dict, int]:
-    sentry_config: Dict[str, str] = get_instance_settings(["SENTRY_AUTH_TOKEN", "SENTRY_ORGANIZATION"])
+def get_sentry_stats(start_time: str, end_time: str) -> tuple[dict, int]:
+    sentry_config: dict[str, str] = get_instance_settings(["SENTRY_AUTH_TOKEN", "SENTRY_ORGANIZATION"])
 
     org_slug = sentry_config.get("SENTRY_ORGANIZATION")
     token = sentry_config.get("SENTRY_AUTH_TOKEN")
@@ -41,9 +41,9 @@ def get_sentry_stats(start_time: str, end_time: str) -> Tuple[dict, int]:
 
 
 def get_tagged_issues_stats(
-    start_time: str, end_time: str, tags: Dict[str, str], target_issues: List[str]
-) -> Dict[str, Any]:
-    sentry_config: Dict[str, str] = get_instance_settings(["SENTRY_AUTH_TOKEN", "SENTRY_ORGANIZATION"])
+    start_time: str, end_time: str, tags: dict[str, str], target_issues: list[str]
+) -> dict[str, Any]:
+    sentry_config: dict[str, str] = get_instance_settings(["SENTRY_AUTH_TOKEN", "SENTRY_ORGANIZATION"])
 
     org_slug = sentry_config.get("SENTRY_ORGANIZATION")
     token = sentry_config.get("SENTRY_AUTH_TOKEN")
@@ -58,7 +58,7 @@ def get_tagged_issues_stats(
     for tag, value in tags.items():
         query += f" {tag}:{value}"
 
-    params: Dict[str, Union[list, str]] = {
+    params: dict[str, Union[list, str]] = {
         "start": start_time,
         "end": end_time,
         "sort": "freq",
@@ -89,8 +89,8 @@ def get_stats_for_timerange(
     base_end_time: str,
     target_start_time: str,
     target_end_time: str,
-    tags: Optional[Dict[str, str]] = None,
-) -> Tuple[int, int]:
+    tags: Optional[dict[str, str]] = None,
+) -> tuple[int, int]:
     base_counts, base_total_count = get_sentry_stats(base_start_time, base_end_time)
     target_counts, target_total_count = get_sentry_stats(target_start_time, target_end_time)
 

ee/api/subscription.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import Any
 
 import jwt
 from django.db.models import QuerySet
@@ -67,7 +67,7 @@ def validate(self, attrs):
 
         return attrs
 
-    def create(self, validated_data: Dict, *args: Any, **kwargs: Any) -> Subscription:
+    def create(self, validated_data: dict, *args: Any, **kwargs: Any) -> Subscription:
         request = self.context["request"]
         validated_data["team_id"] = self.context["team_id"]
         validated_data["created_by"] = request.user

ee/api/test/base.py

@@ -1,5 +1,5 @@
 import datetime
-from typing import Dict, Optional, cast
+from typing import Optional, cast
 
 from zoneinfo import ZoneInfo
 
@@ -20,7 +20,7 @@ class LicensedTestMixin:
     def license_required_response(
         self,
         message: str = "This feature is part of the premium PostHog offering. Self-hosted licenses are no longer available for purchase. Please contact [email protected] to discuss options.",
-    ) -> Dict[str, Optional[str]]:
+    ) -> dict[str, Optional[str]]:
         return {
             "type": "server_error",
             "code": "payment_required",

ee/api/test/fixtures/available_product_features.py

@@ -1,6 +1,6 @@
-from typing import Any, Dict, List
+from typing import Any
 
-AVAILABLE_PRODUCT_FEATURES: List[Dict[str, Any]] = [
+AVAILABLE_PRODUCT_FEATURES: list[dict[str, Any]] = [
     {
         "description": "Create playlists of certain session recordings to easily find and watch them again in the future.",
         "key": "recordings_playlists",

ee/api/test/test_authentication.py

@@ -364,7 +364,6 @@ def test_can_login_with_saml(self):
 
         with open(
             os.path.join(CURRENT_FOLDER, "fixtures/saml_login_response"),
-            "r",
             encoding="utf_8",
         ) as f:
             saml_response = f.read()
@@ -407,7 +406,6 @@ def test_saml_jit_provisioning_and_assertion_with_different_attribute_names(self
 
         with open(
             os.path.join(CURRENT_FOLDER, "fixtures/saml_login_response_alt_attribute_names"),
-            "r",
             encoding="utf_8",
         ) as f:
             saml_response = f.read()
@@ -474,7 +472,6 @@ def test_cannot_login_with_improperly_signed_payload(self):
 
         with open(
             os.path.join(CURRENT_FOLDER, "fixtures/saml_login_response"),
-            "r",
             encoding="utf_8",
         ) as f:
             saml_response = f.read()
@@ -514,7 +511,6 @@ def test_cannot_signup_with_saml_if_jit_provisioning_is_disabled(self):
 
         with open(
             os.path.join(CURRENT_FOLDER, "fixtures/saml_login_response"),
-            "r",
             encoding="utf_8",
         ) as f:
             saml_response = f.read()
@@ -552,7 +548,6 @@ def test_cannot_create_account_without_first_name_in_payload(self):
 
         with open(
             os.path.join(CURRENT_FOLDER, "fixtures/saml_login_response_no_first_name"),
-            "r",
             encoding="utf_8",
         ) as f:
             saml_response = f.read()
@@ -594,7 +589,6 @@ def test_cannot_login_with_saml_on_unverified_domain(self):
 
         with open(
             os.path.join(CURRENT_FOLDER, "fixtures/saml_login_response"),
-            "r",
             encoding="utf_8",
         ) as f:
             saml_response = f.read()
@@ -683,7 +677,6 @@ def test_cannot_use_saml_without_enterprise_license(self):
 
         with open(
             os.path.join(CURRENT_FOLDER, "fixtures/saml_login_response"),
-            "r",
             encoding="utf_8",
         ) as f:
             saml_response = f.read()

ee/api/test/test_billing.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Any, Dict, List
+from typing import Any
 from unittest.mock import MagicMock, patch
 from uuid import uuid4
 from zoneinfo import ZoneInfo
@@ -22,7 +22,7 @@
 from posthog.test.base import APIBaseTest, _create_event, flush_persons_and_events
 
 
-def create_billing_response(**kwargs) -> Dict[str, Any]:
+def create_billing_response(**kwargs) -> dict[str, Any]:
     data: Any = {"license": {"type": "cloud"}}
     data.update(kwargs)
     return data
@@ -106,7 +106,7 @@ def create_billing_customer(**kwargs) -> CustomerInfo:
     return data
 
 
-def create_billing_products_response(**kwargs) -> Dict[str, List[CustomerProduct]]:
+def create_billing_products_response(**kwargs) -> dict[str, list[CustomerProduct]]:
     data: Any = {
         "products": [
             CustomerProduct(

ee/api/test/test_capture.py

@@ -68,26 +68,26 @@ def test_produce_to_kafka(self, kafka_produce):
         self.assertEqual(event2_data["properties"]["distinct_id"], "id2")
 
         # Make sure we're producing data correctly in the way the plugin server expects
-        self.assertEquals(type(kafka_produce_call1["data"]["distinct_id"]), str)
-        self.assertEquals(type(kafka_produce_call2["data"]["distinct_id"]), str)
+        self.assertEqual(type(kafka_produce_call1["data"]["distinct_id"]), str)
+        self.assertEqual(type(kafka_produce_call2["data"]["distinct_id"]), str)
 
         self.assertIn(type(kafka_produce_call1["data"]["ip"]), [str, type(None)])
         self.assertIn(type(kafka_produce_call2["data"]["ip"]), [str, type(None)])
 
-        self.assertEquals(type(kafka_produce_call1["data"]["site_url"]), str)
-        self.assertEquals(type(kafka_produce_call2["data"]["site_url"]), str)
+        self.assertEqual(type(kafka_produce_call1["data"]["site_url"]), str)
+        self.assertEqual(type(kafka_produce_call2["data"]["site_url"]), str)
 
-        self.assertEquals(type(kafka_produce_call1["data"]["token"]), str)
-        self.assertEquals(type(kafka_produce_call2["data"]["token"]), str)
+        self.assertEqual(type(kafka_produce_call1["data"]["token"]), str)
+        self.assertEqual(type(kafka_produce_call2["data"]["token"]), str)
 
-        self.assertEquals(type(kafka_produce_call1["data"]["sent_at"]), str)
-        self.assertEquals(type(kafka_produce_call2["data"]["sent_at"]), str)
+        self.assertEqual(type(kafka_produce_call1["data"]["sent_at"]), str)
+        self.assertEqual(type(kafka_produce_call2["data"]["sent_at"]), str)
 
-        self.assertEquals(type(event1_data["properties"]), dict)
-        self.assertEquals(type(event2_data["properties"]), dict)
+        self.assertEqual(type(event1_data["properties"]), dict)
+        self.assertEqual(type(event2_data["properties"]), dict)
 
-        self.assertEquals(type(kafka_produce_call1["data"]["uuid"]), str)
-        self.assertEquals(type(kafka_produce_call2["data"]["uuid"]), str)
+        self.assertEqual(type(kafka_produce_call1["data"]["uuid"]), str)
+        self.assertEqual(type(kafka_produce_call2["data"]["uuid"]), str)
 
     @patch("posthog.kafka_client.client._KafkaProducer.produce")
     def test_capture_event_with_uuid_in_payload(self, kafka_produce):