refactor(taxonomy): Add project field to taxonomy models (#26521)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

plugin-server/src/types.ts

@@ -1103,6 +1103,7 @@ export interface EventDefinitionType {
     volume_30_day: number | null
     query_usage_30_day: number | null
     team_id: number
+    project_id: number | null
     last_seen_at: string // DateTime
     created_at: string // DateTime
 }
@@ -1142,6 +1143,7 @@ export interface PropertyDefinitionType {
     volume_30_day: number | null
     query_usage_30_day: number | null
     team_id: number
+    project_id: number | null
     property_type?: PropertyType
     type: PropertyDefinitionTypeEnum
     group_type_index: number | null
@@ -1152,6 +1154,7 @@ export interface EventPropertyType {
     event: string
     property: string
     team_id: number
+    project_id: number | null
 }
 
 export type PluginFunction = 'onEvent' | 'processEvent' | 'pluginTask'

posthog/api/test/__snapshots__/test_api_docs.ambr

@@ -34,7 +34,6 @@
     '/home/runner/work/posthog/posthog/posthog/api/event.py: Warning [EventViewSet]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. <int:id>) or annotating the parameter type with @extend_schema. Defaulting to "string".',
     '/home/runner/work/posthog/posthog/posthog/api/event.py: Warning [EventViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. <int:project_id>) or annotating the parameter type with @extend_schema. Defaulting to "string".',
     "/home/runner/work/posthog/posthog/posthog/api/event_definition.py: Error [EventDefinitionViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AnonymousUser' object has no attribute 'organization')",
-    '/home/runner/work/posthog/posthog/posthog/api/event_definition.py: Warning [EventDefinitionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.event_definition.EventDefinition" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".',
     '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet > ExportedAssetSerializer]: unable to resolve type hint for function "filename". Consider using a type hint or @extend_schema_field. Defaulting to string.',
     '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet > ExportedAssetSerializer]: unable to resolve type hint for function "has_content". Consider using a type hint or @extend_schema_field. Defaulting to string.',
     '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.exported_asset.ExportedAsset" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".',
@@ -59,7 +58,6 @@
     '/home/runner/work/posthog/posthog/posthog/api/project.py: Warning [ProjectViewSet > ProjectBackwardCompatSerializer]: could not resolve field on model <class \'posthog.models.project.Project\'> with path "person_on_events_querying_enabled". This is likely a custom field that does some unknown magic. Maybe consider annotating the field/property? Defaulting to "string". (Exception: Project has no field named \'person_on_events_querying_enabled\')',
     '/home/runner/work/posthog/posthog/posthog/api/project.py: Warning [ProjectViewSet > ProjectBackwardCompatSerializer]: unable to resolve type hint for function "get_product_intents". Consider using a type hint or @extend_schema_field. Defaulting to string.',
     "/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Error [PropertyDefinitionViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AnonymousUser' object has no attribute 'organization')",
-    '/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Warning [PropertyDefinitionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.property_definition.PropertyDefinition" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".',
     '/home/runner/work/posthog/posthog/posthog/api/proxy_record.py: Warning [ProxyRecordViewset]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. <int:id>) or annotating the parameter type with @extend_schema. Defaulting to "string".',
     '/home/runner/work/posthog/posthog/posthog/api/proxy_record.py: Warning [ProxyRecordViewset]: could not derive type of path parameter "organization_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. <int:organization_id>) or annotating the parameter type with @extend_schema. Defaulting to "string".',
     '/home/runner/work/posthog/posthog/posthog/api/query.py: Error [QueryViewSet]: unable to guess serializer. This is graceful fallback handling for APIViews. Consider using GenericAPIView as view base class, if view is under your control. Either way you may want to add a serializer_class (or method). Ignoring view for now.',

posthog/migrations/0528_project_field_in_taxonomy.py

@@ -0,0 +1,89 @@
+# Generated by Django 4.2.15 on 2024-10-29 12:15
+
+from django.db import migrations, models
+import django.db.models.deletion
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db.models.expressions import F, OrderBy
+from django.db.models.functions.comparison import Coalesce
+
+
+class Migration(migrations.Migration):
+    atomic = False  # Added to support concurrent index creation
+    dependencies = [("posthog", "0527_project_name_sync")]
+
+    operations = [
+        migrations.RunSQL(
+            """
+            -- Add field project to eventdefinition
+            ALTER TABLE "posthog_eventdefinition" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_eventdefinit_project_id_f93fcbb0_fk_posthog_p" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED;
+            SET CONSTRAINTS "posthog_eventdefinit_project_id_f93fcbb0_fk_posthog_p" IMMEDIATE;
+            -- Add field project to eventproperty
+            ALTER TABLE "posthog_eventproperty" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_eventproperty_project_id_dd2337d2_fk_posthog_project_id" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED;
+            SET CONSTRAINTS "posthog_eventproperty_project_id_dd2337d2_fk_posthog_project_id" IMMEDIATE;
+            -- Add field project to propertydefinition
+            ALTER TABLE "posthog_propertydefinition" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_propertydefi_project_id_d3eb982d_fk_posthog_p" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED;
+            SET CONSTRAINTS "posthog_propertydefi_project_id_d3eb982d_fk_posthog_p" IMMEDIATE;""",
+            reverse_sql="""
+            ALTER TABLE "posthog_eventdefinition" DROP COLUMN IF EXISTS "project_id";
+            ALTER TABLE "posthog_eventproperty" DROP COLUMN IF EXISTS "project_id";
+            ALTER TABLE "posthog_propertydefinition" DROP COLUMN IF EXISTS "project_id";""",
+            state_operations=[
+                migrations.AddField(
+                    model_name="eventdefinition",
+                    name="project",
+                    field=models.ForeignKey(
+                        null=True, on_delete=django.db.models.deletion.CASCADE, to="posthog.project"
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="eventproperty",
+                    name="project",
+                    field=models.ForeignKey(
+                        null=True, on_delete=django.db.models.deletion.CASCADE, to="posthog.project"
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="propertydefinition",
+                    name="project",
+                    field=models.ForeignKey(
+                        null=True, on_delete=django.db.models.deletion.CASCADE, to="posthog.project"
+                    ),
+                ),
+            ],
+        ),
+        AddIndexConcurrently(
+            model_name="eventdefinition",
+            index=models.Index(fields=["project"], name="posthog_eve_proj_id_f93fcbb0"),
+        ),
+        AddIndexConcurrently(
+            model_name="propertydefinition",
+            index=models.Index(fields=["project"], name="posthog_prop_proj_id_d3eb982d"),
+        ),
+        AddIndexConcurrently(
+            model_name="propertydefinition",
+            index=models.Index(
+                F("project_id"),
+                F("type"),
+                Coalesce(F("group_type_index"), -1),
+                OrderBy(F("query_usage_30_day"), descending=True, nulls_last=True),
+                OrderBy(F("name")),
+                name="index_property_def_query_proj",
+            ),
+        ),
+        AddIndexConcurrently(
+            model_name="propertydefinition",
+            index=models.Index(fields=["project_id", "type", "is_numerical"], name="posthog_pro_project_3583d2_idx"),
+        ),
+        AddIndexConcurrently(
+            model_name="eventproperty",
+            index=models.Index(fields=["project"], name="posthog_eve_proj_id_dd2337d2"),
+        ),
+        AddIndexConcurrently(
+            model_name="eventproperty",
+            index=models.Index(fields=["project", "event"], name="posthog_eve_proj_id_22de03_idx"),
+        ),
+        AddIndexConcurrently(
+            model_name="eventproperty",
+            index=models.Index(fields=["project", "property"], name="posthog_eve_proj_id_26dbfb_idx"),
+        ),
+    ]

posthog/migrations/max_migration.txt

@@ -1 +1 @@
-0527_project_name_sync
+0528_project_field_in_taxonomy

posthog/models/event_definition.py

@@ -13,6 +13,7 @@ class EventDefinition(UUIDModel):
         related_name="event_definitions",
         related_query_name="team",
     )
+    project = models.ForeignKey("Project", on_delete=models.CASCADE, null=True)
     name = models.CharField(max_length=400)
     created_at = models.DateTimeField(default=timezone.now, null=True)
     last_seen_at = models.DateTimeField(default=None, null=True)
@@ -28,11 +29,13 @@ class EventDefinition(UUIDModel):
     class Meta:
         unique_together = ("team", "name")
         indexes = [
+            # Index on project_id foreign key
+            models.Index(fields=["project"], name="posthog_eve_proj_id_f93fcbb0"),
             GinIndex(
                 name="index_event_definition_name",
                 fields=["name"],
                 opclasses=["gin_trgm_ops"],
-            )  # To speed up DB-based fuzzy searching
+            ),  # To speed up DB-based fuzzy searching
         ]
 
     def __str__(self) -> str:

posthog/models/event_property.py

@@ -6,6 +6,7 @@
 
 class EventProperty(models.Model):
     team = models.ForeignKey(Team, on_delete=models.CASCADE)
+    project = models.ForeignKey("Project", on_delete=models.CASCADE, null=True)
     event = models.CharField(max_length=400, null=False)
     property = models.CharField(max_length=400, null=False)
 
@@ -17,8 +18,12 @@ class Meta:
             )
         ]
         indexes = [
+            # Index on project_id foreign key
+            models.Index(fields=["project"], name="posthog_eve_proj_id_dd2337d2"),
             models.Index(fields=["team", "event"]),
+            models.Index(fields=["project", "event"], name="posthog_eve_proj_id_22de03_idx"),
             models.Index(fields=["team", "property"]),
+            models.Index(fields=["project", "property"], name="posthog_eve_proj_id_26dbfb_idx"),
         ]
 
     __repr__ = sane_repr("event", "property", "team_id")

posthog/models/property_definition.py

@@ -43,6 +43,7 @@ class Type(models.IntegerChoices):
         related_name="property_definitions",
         related_query_name="team",
     )
+    project = models.ForeignKey("Project", on_delete=models.CASCADE, null=True)
     name = models.CharField(max_length=400)
     is_numerical = models.BooleanField(
         default=False
@@ -69,6 +70,8 @@ class Type(models.IntegerChoices):
 
     class Meta:
         indexes = [
+            # Index on project_id foreign key
+            models.Index(fields=["project"], name="posthog_prop_proj_id_d3eb982d"),
             # This indexes the query in api/property_definition.py
             # :KLUDGE: django ORM typing is off here
             models.Index(
@@ -79,9 +82,18 @@ class Meta:
                 F("name").asc(),
                 name="index_property_def_query",
             ),
+            models.Index(
+                F("project_id"),
+                F("type"),
+                Coalesce(F("group_type_index"), -1),
+                F("query_usage_30_day").desc(nulls_last=True),
+                F("name").asc(),
+                name="index_property_def_query_proj",
+            ),
             # creates an index pganalyze identified as missing
             # https://app.pganalyze.com/servers/i35ydkosi5cy5n7tly45vkjcqa/checks/index_advisor/missing_index/15282978
             models.Index(fields=["team_id", "type", "is_numerical"]),
+            models.Index(fields=["project_id", "type", "is_numerical"], name="posthog_pro_project_3583d2_idx"),
             GinIndex(
                 name="index_property_definition_name",
                 fields=["name"],

rust/common/types/src/event.rs

@@ -39,7 +39,7 @@ pub enum PersonMode {
 pub struct ClickHouseEvent {
     pub uuid: Uuid,
     pub team_id: i32,
-    pub project_id: i32,
+    pub project_id: i64,
     pub event: String,
     pub distinct_id: String,
     #[serde(skip_serializing_if = "Option::is_none")]