chore(deps): Upgrade to Django 4.1 (#18830)

* Upgrade dependencies
* Fix middleware error

Among
https://docs.djangoproject.com/en/4.2/releases/4.0/#features-removed-in-4-0

* Switch TaggedItem tests to assert ValidationError

Because full_clean validates since Django 4.1, see
https://docs.djangoproject.com/en/4.2/releases/4.1/#validation-of-constraints

* Remove type: ignore comments

* Set CSRF_TRUSTED_ORIGINS

* Update access.py

Co-authored-by: Paul D'Ambra <[email protected]>

ee/api/test/__snapshots__/test_organization_resource_access.ambr

@@ -78,7 +78,7 @@
 ---
 # name: TestOrganizationResourceAccessAPI.test_list_organization_resource_access_is_not_nplus1.11
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_organizationmembership"
   WHERE ("posthog_organizationmembership"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid
          AND "posthog_organizationmembership"."user_id" = 2)
@@ -160,7 +160,7 @@
 ---
 # name: TestOrganizationResourceAccessAPI.test_list_organization_resource_access_is_not_nplus1.3
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_organizationmembership"
   WHERE ("posthog_organizationmembership"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid
          AND "posthog_organizationmembership"."user_id" = 2)

posthog/api/test/__snapshots__/test_decide.ambr

@@ -137,7 +137,7 @@
 ---
 # name: TestDecide.test_decide_doesnt_error_out_when_database_is_down.3
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_grouptypemapping"
   WHERE "posthog_grouptypemapping"."team_id" = 2
   LIMIT 1 /*controller='team-detail',route='api/projects/%28%3FP%3Cid%3E%5B%5E/.%5D%2B%29/%3F%24'*/

posthog/api/test/__snapshots__/test_organization_feature_flag.ambr

@@ -347,7 +347,7 @@
 ---
 # name: TestOrganizationFeatureFlagCopy.test_copy_feature_flag_create_new.2
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_organizationmembership"
   WHERE ("posthog_organizationmembership"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid
          AND "posthog_organizationmembership"."user_id" = 2)
@@ -1539,7 +1539,7 @@
 ---
 # name: TestOrganizationFeatureFlagCopy.test_copy_feature_flag_create_new.8
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_featureflag"
   WHERE (NOT "posthog_featureflag"."deleted"
          AND "posthog_featureflag"."key" = 'copied-flag-key'
@@ -1626,7 +1626,7 @@
 ---
 # name: TestOrganizationFeatureFlagGet.test_get_feature_flag_success.2
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_organizationmembership"
   WHERE ("posthog_organizationmembership"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid
          AND "posthog_organizationmembership"."user_id" = 2)

posthog/api/test/__snapshots__/test_plugin.ambr

@@ -201,7 +201,7 @@
 ---
 # name: TestPluginAPI.test_listing_plugins_is_not_nplus1.15
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_organizationmembership"
   WHERE ("posthog_organizationmembership"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid
          AND "posthog_organizationmembership"."user_id" = 2)
@@ -381,7 +381,7 @@
 ---
 # name: TestPluginAPI.test_listing_plugins_is_not_nplus1.21
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_organizationmembership"
   WHERE ("posthog_organizationmembership"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid
          AND "posthog_organizationmembership"."user_id" = 2)
@@ -459,7 +459,7 @@
 ---
 # name: TestPluginAPI.test_listing_plugins_is_not_nplus1.3
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_organizationmembership"
   WHERE ("posthog_organizationmembership"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid
          AND "posthog_organizationmembership"."user_id" = 2)
@@ -570,7 +570,7 @@
 ---
 # name: TestPluginAPI.test_listing_plugins_is_not_nplus1.9
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_organizationmembership"
   WHERE ("posthog_organizationmembership"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid
          AND "posthog_organizationmembership"."user_id" = 2)

posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr

@@ -961,6 +961,9 @@
               OR "posthog_dashboardtile"."insight_id" IS NULL)
          AND "posthog_dashboardtile"."dashboard_id" = 2
          AND "posthog_dashboardtile"."dashboard_id" = 2
+         AND NOT ("posthog_dashboard"."deleted"
+                  AND "posthog_dashboardtile"."deleted"
+                  AND "posthog_dashboardtile"."deleted" IS NOT NULL)
          AND (NOT "posthog_dashboarditem"."deleted"
               OR "posthog_dashboardtile"."insight_id" IS NULL))
   ORDER BY "posthog_dashboarditem"."order" ASC /*controller='project_dashboards-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%28%3FP%3Cpk%3E%5B%5E/.%5D%2B%29/%3F%24'*/
@@ -7351,6 +7354,9 @@
               OR "posthog_dashboardtile"."insight_id" IS NULL)
          AND "posthog_dashboardtile"."dashboard_id" = 2
          AND "posthog_dashboardtile"."dashboard_id" = 2
+         AND NOT ("posthog_dashboard"."deleted"
+                  AND "posthog_dashboardtile"."deleted"
+                  AND "posthog_dashboardtile"."deleted" IS NOT NULL)
          AND (NOT "posthog_dashboarditem"."deleted"
               OR "posthog_dashboardtile"."insight_id" IS NULL))
   ORDER BY "posthog_dashboarditem"."order" ASC /*controller='project_dashboards-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%28%3FP%3Cpk%3E%5B%5E/.%5D%2B%29/%3F%24'*/
@@ -10019,6 +10025,9 @@
               OR "posthog_dashboardtile"."insight_id" IS NULL)
          AND "posthog_dashboardtile"."dashboard_id" = 2
          AND "posthog_dashboardtile"."dashboard_id" = 2
+         AND NOT ("posthog_dashboard"."deleted"
+                  AND "posthog_dashboardtile"."deleted"
+                  AND "posthog_dashboardtile"."deleted" IS NOT NULL)
          AND (NOT "posthog_dashboarditem"."deleted"
               OR "posthog_dashboardtile"."insight_id" IS NULL))
   ORDER BY "posthog_dashboarditem"."order" ASC /*controller='project_dashboards-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%28%3FP%3Cpk%3E%5B%5E/.%5D%2B%29/%3F%24'*/

posthog/api/test/test_organization.py

@@ -154,4 +154,4 @@ async def acreate_organization(name: str) -> Organization:
     could use either the api, or django admin to create, to get better parity
     with real world scenarios.
     """
-    return await sync_to_async(create_organization)(name)  # type: ignore
+    return await sync_to_async(create_organization)(name)

posthog/api/test/test_team.py

@@ -736,4 +736,4 @@ async def acreate_team(organization: Organization, name: str = "Test team") -> T
     could use either the api, or django admin to create, to get better parity
     with real world  scenarios.
     """
-    return await sync_to_async(create_team)(organization, name=name)  # type: ignore
+    return await sync_to_async(create_team)(organization, name=name)

posthog/middleware.py

@@ -362,7 +362,7 @@ def __init__(self, get_response):
                 # Some middlewares raise MiddlewareNotUsed if they are not
                 # needed. In this case we want to avoid the default middlewares
                 # being used.
-                middlewares.append(middleware_class(get_response=None))
+                middlewares.append(middleware_class(get_response=get_response))
             except MiddlewareNotUsed:
                 pass
 

posthog/models/filters/test/__snapshots__/test_filter.ambr

@@ -348,7 +348,7 @@
 ---
 # name: TestDjangoPropertiesToQ.test_person_relative_date_parsing_with_invalid_date
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_person"
   INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id")
   WHERE ("posthog_persondistinctid"."distinct_id" = 'example_id'
@@ -359,7 +359,7 @@
 ---
 # name: TestDjangoPropertiesToQ.test_person_relative_date_parsing_with_invalid_date.1
   '
-  SELECT (1) AS "a"
+  SELECT 1 AS "a"
   FROM "posthog_person"
   INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id")
   WHERE ("posthog_persondistinctid"."distinct_id" = 'example_id'

posthog/models/test/test_tagged_item_model.py

@@ -1,5 +1,4 @@
 from django.core.exceptions import ValidationError
-from django.db import IntegrityError
 
 from posthog.models import Action, Dashboard, DashboardTile, Insight, Tag, TaggedItem
 from posthog.test.base import BaseTest
@@ -28,7 +27,7 @@ def test_uniqueness_constraint_dashboard(self):
         tag = Tag.objects.create(name="tag", team_id=self.team.id)
 
         TaggedItem.objects.create(dashboard_id=dashboard.id, tag_id=tag.id)
-        with self.assertRaises(IntegrityError):
+        with self.assertRaises(ValidationError):
             TaggedItem.objects.create(dashboard_id=dashboard.id, tag_id=tag.id)
 
     def test_uniqueness_constraint_insight(self):
@@ -38,7 +37,7 @@ def test_uniqueness_constraint_insight(self):
         tag = Tag.objects.create(name="tag", team_id=self.team.id)
 
         TaggedItem.objects.create(insight_id=insight.id, tag_id=tag.id)
-        with self.assertRaises(IntegrityError):
+        with self.assertRaises(ValidationError):
             TaggedItem.objects.create(insight_id=insight.id, tag_id=tag.id)
 
     def test_uniqueness_constraint_event_definition(self):
@@ -53,7 +52,7 @@ def test_uniqueness_constraint_event_definition(self):
             tag = Tag.objects.create(name="tag", team_id=self.team.id)
 
             TaggedItem.objects.create(event_definition_id=event_definition.id, tag_id=tag.id)
-            with self.assertRaises(IntegrityError):
+            with self.assertRaises(ValidationError):
                 TaggedItem.objects.create(event_definition_id=event_definition.id, tag_id=tag.id)
 
     def test_uniqueness_constraint_property_definition(self):
@@ -68,13 +67,13 @@ def test_uniqueness_constraint_property_definition(self):
             tag = Tag.objects.create(name="tag", team_id=self.team.id)
 
             TaggedItem.objects.create(property_definition_id=property_definition.id, tag_id=tag.id)
-            with self.assertRaises(IntegrityError):
+            with self.assertRaises(ValidationError):
                 TaggedItem.objects.create(property_definition_id=property_definition.id, tag_id=tag.id)
 
     def test_uniqueness_constraint_action(self):
         action = Action.objects.create(team=self.team, name="enterprise property")
         tag = Tag.objects.create(name="tag", team_id=self.team.id)
 
         TaggedItem.objects.create(action_id=action.id, tag_id=tag.id)
-        with self.assertRaises(IntegrityError):
+        with self.assertRaises(ValidationError):
             TaggedItem.objects.create(action_id=action.id, tag_id=tag.id)

posthog/settings/access.py

@@ -25,6 +25,7 @@
 if get_from_env("DISABLE_SECURE_SSL_REDIRECT", False, type_cast=str_to_bool):
     SECURE_SSL_REDIRECT = False
 
+CSRF_TRUSTED_ORIGINS = [os.getenv("SITE_URL", "http://localhost:8000").rstrip("/")]
 
 # Proxy settings
 IS_BEHIND_PROXY = get_from_env("IS_BEHIND_PROXY", False, type_cast=str_to_bool)

posthog/temporal/batch_exports/batch_exports.py

@@ -513,7 +513,7 @@ async def create_export_run(inputs: CreateBatchExportRunInputs) -> str:
     # 'sync_to_async' type hints are fixed in asgiref>=3.4.1
     # But one of our dependencies is pinned to asgiref==3.3.2.
     # Remove these comments once we upgrade.
-    run = await sync_to_async(create_batch_export_run)(  # type: ignore
+    run = await sync_to_async(create_batch_export_run)(
         batch_export_id=uuid.UUID(inputs.batch_export_id),
         data_interval_start=inputs.data_interval_start,
         data_interval_end=inputs.data_interval_end,
@@ -542,7 +542,7 @@ async def update_export_run_status(inputs: UpdateBatchExportRunStatusInputs):
         run_id=uuid.UUID(inputs.id),
         status=inputs.status,
         latest_error=inputs.latest_error,
-    )  # type: ignore
+    )
 
     if batch_export_run.status == "Failed":
         logger.error("BatchExport failed with error: %s", batch_export_run.latest_error)
@@ -583,7 +583,7 @@ async def create_batch_export_backfill_model(inputs: CreateBatchExportBackfillIn
     # 'sync_to_async' type hints are fixed in asgiref>=3.4.1
     # But one of our dependencies is pinned to asgiref==3.3.2.
     # Remove these comments once we upgrade.
-    run = await sync_to_async(create_batch_export_backfill)(  # type: ignore
+    run = await sync_to_async(create_batch_export_backfill)(
         batch_export_id=uuid.UUID(inputs.batch_export_id),
         start_at=inputs.start_at,
         end_at=inputs.end_at,
@@ -607,7 +607,7 @@ async def update_batch_export_backfill_model_status(inputs: UpdateBatchExportBac
     """Activity that updates the status of an BatchExportRun."""
     backfill = await sync_to_async(update_batch_export_backfill_status)(
         backfill_id=uuid.UUID(inputs.id), status=inputs.status
-    )  # type: ignore
+    )
     logger = await bind_temporal_worker_logger(team_id=backfill.team_id)
 
     if backfill.status == "Failed":

posthog/temporal/data_imports/external_data_job.py

@@ -38,24 +38,24 @@ class CreateExternalDataJobInputs:
 
 @activity.defn
 async def create_external_data_job_model(inputs: CreateExternalDataJobInputs) -> Tuple[str, list[str]]:
-    run = await sync_to_async(create_external_data_job)(  # type: ignore
+    run = await sync_to_async(create_external_data_job)(
         team_id=inputs.team_id,
         external_data_source_id=inputs.external_data_source_id,
         workflow_id=activity.info().workflow_id,
     )
 
-    source = await sync_to_async(ExternalDataSource.objects.get)(  # type: ignore
+    source = await sync_to_async(ExternalDataSource.objects.get)(
         team_id=inputs.team_id, id=inputs.external_data_source_id
     )
 
     # Sync schemas if they have changed
-    await sync_to_async(sync_old_schemas_with_new_schemas)(  # type: ignore
-        list(PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[source.source_type]),  # type: ignore
+    await sync_to_async(sync_old_schemas_with_new_schemas)(
+        list(PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[source.source_type]),
         source_id=inputs.external_data_source_id,
         team_id=inputs.team_id,
     )
 
-    schemas = await sync_to_async(get_active_schemas_for_source_id)(  # type: ignore
+    schemas = await sync_to_async(get_active_schemas_for_source_id)(
         team_id=inputs.team_id, source_id=inputs.external_data_source_id
     )
 
@@ -79,7 +79,7 @@ class UpdateExternalDataJobStatusInputs:
 
 @activity.defn
 async def update_external_data_job_model(inputs: UpdateExternalDataJobStatusInputs) -> None:
-    await sync_to_async(update_external_job_status)(  # type: ignore
+    await sync_to_async(update_external_job_status)(
         run_id=uuid.UUID(inputs.id),
         status=inputs.status,
         latest_error=inputs.latest_error,
@@ -101,7 +101,7 @@ class ValidateSchemaInputs:
 
 @activity.defn
 async def validate_schema_activity(inputs: ValidateSchemaInputs) -> None:
-    await sync_to_async(validate_schema_and_update_table)(  # type: ignore
+    await sync_to_async(validate_schema_and_update_table)(
         run_id=inputs.run_id,
         team_id=inputs.team_id,
         schemas=inputs.schemas,
@@ -129,7 +129,7 @@ class ExternalDataJobInputs:
 
 @activity.defn
 async def run_external_data_job(inputs: ExternalDataJobInputs) -> None:
-    model: ExternalDataJob = await sync_to_async(get_external_data_job)(  # type: ignore
+    model: ExternalDataJob = await sync_to_async(get_external_data_job)(
         team_id=inputs.team_id,
         run_id=inputs.run_id,
     )

posthog/temporal/data_imports/pipelines/stripe/helpers.py

@@ -74,6 +74,6 @@ async def stripe_get_data(
         api_key=api_key,
         created={"gte": start_date, "lt": end_date},
         limit=100,
-        **kwargs,  # type: ignore
+        **kwargs,
     )
     return dict(resource_dict)

posthog/temporal/tests/conftest.py

@@ -40,21 +40,21 @@ def team(organization):
 @pytest_asyncio.fixture
 async def aorganization():
     name = f"BatchExportsTestOrg-{random.randint(1, 99999)}"
-    org = await sync_to_async(Organization.objects.create)(name=name)  # type: ignore
+    org = await sync_to_async(Organization.objects.create)(name=name)
 
     yield org
 
-    await sync_to_async(org.delete)()  # type: ignore
+    await sync_to_async(org.delete)()
 
 
 @pytest_asyncio.fixture
 async def ateam(aorganization):
     name = f"BatchExportsTestTeam-{random.randint(1, 99999)}"
-    team = await sync_to_async(Team.objects.create)(organization=aorganization, name=name)  # type: ignore
+    team = await sync_to_async(Team.objects.create)(organization=aorganization, name=name)
 
     yield team
 
-    await sync_to_async(team.delete)()  # type: ignore
+    await sync_to_async(team.delete)()
 
 
 @pytest.fixture