From dd0e0d792f6585b28200339858eeac209c4b8fc5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 29 Oct 2024 14:38:56 +0100 Subject: [PATCH 1/9] refactor(taxonomy): Add project field to taxonomy models --- plugin-server/src/types.ts | 3 + .../0526_project_field_in_taxonomy.py | 66 +++++++++++++++++++ posthog/migrations/max_migration.txt | 2 +- posthog/models/event_definition.py | 1 + posthog/models/event_property.py | 1 + posthog/models/property_definition.py | 1 + rust/property-defs-rs/src/types.rs | 14 ++-- 7 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 posthog/migrations/0526_project_field_in_taxonomy.py diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index e8682da3b337b..47ac3764a3528 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -1103,6 +1103,7 @@ export interface EventDefinitionType { volume_30_day: number | null query_usage_30_day: number | null team_id: number + project_id: number | null last_seen_at: string // DateTime created_at: string // DateTime } @@ -1142,6 +1143,7 @@ export interface PropertyDefinitionType { volume_30_day: number | null query_usage_30_day: number | null team_id: number + project_id: number | null property_type?: PropertyType type: PropertyDefinitionTypeEnum group_type_index: number | null @@ -1152,6 +1154,7 @@ export interface EventPropertyType { event: string property: string team_id: number + project_id: number | null } export type PluginFunction = 'onEvent' | 'processEvent' | 'pluginTask' diff --git a/posthog/migrations/0526_project_field_in_taxonomy.py b/posthog/migrations/0526_project_field_in_taxonomy.py new file mode 100644 index 0000000000000..1d0909987c1a4 --- /dev/null +++ b/posthog/migrations/0526_project_field_in_taxonomy.py @@ -0,0 +1,66 @@ +# Generated by Django 4.2.15 on 2024-10-29 12:15 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + atomic = False # Added to support concurrent index creation + dependencies = [("posthog", "0525_hog_function_transpiled")] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.AddField( + model_name="eventdefinition", + name="project", + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to="posthog.project" + ), + ), + migrations.AddField( + model_name="eventproperty", + name="project", + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to="posthog.project" + ), + ), + migrations.AddField( + model_name="propertydefinition", + name="project", + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to="posthog.project" + ), + ), + ], + database_operations=[ + migrations.RunSQL( + """ + -- Add field project to eventdefinition + ALTER TABLE "posthog_eventdefinition" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_eventdefinit_project_id_f93fcbb0_fk_posthog_p" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED; + SET CONSTRAINTS "posthog_eventdefinit_project_id_f93fcbb0_fk_posthog_p" IMMEDIATE; + -- Add field project to eventproperty + ALTER TABLE "posthog_eventproperty" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_eventproperty_project_id_dd2337d2_fk_posthog_project_id" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED; + SET CONSTRAINTS "posthog_eventproperty_project_id_dd2337d2_fk_posthog_project_id" IMMEDIATE; + -- Add field project to propertydefinition + ALTER TABLE "posthog_propertydefinition" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_propertydefi_project_id_d3eb982d_fk_posthog_p" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED; + SET CONSTRAINTS "posthog_propertydefi_project_id_d3eb982d_fk_posthog_p" IMMEDIATE;""", + reverse_sql=""" + ALTER TABLE "posthog_eventdefinition" DROP COLUMN IF EXISTS "project_id"; + ALTER TABLE "posthog_eventproperty" DROP COLUMN IF EXISTS "project_id"; + ALTER TABLE "posthog_propertydefinition" DROP COLUMN IF EXISTS "project_id";""", + ), + # We add CONCURRENTLY to the CREATE INDEX commands + migrations.RunSQL( + """ + CREATE INDEX CONCURRENTLY "posthog_eventdefinition_project_id_f93fcbb0" ON "posthog_eventdefinition" ("project_id"); + CREATE INDEX CONCURRENTLY "posthog_eventproperty_project_id_dd2337d2" ON "posthog_eventproperty" ("project_id"); + CREATE INDEX CONCURRENTLY "posthog_propertydefinition_project_id_d3eb982d" ON "posthog_propertydefinition" ("project_id");""", + reverse_sql=""" + DROP INDEX IF EXISTS "posthog_eventdefinition_project_id_f93fcbb0"; + DROP INDEX IF EXISTS "posthog_eventproperty_project_id_dd2337d2"; + DROP INDEX IF EXISTS "posthog_propertydefinition_project_id_d3eb982d";""", + ), + ], + ) + ] diff --git a/posthog/migrations/max_migration.txt b/posthog/migrations/max_migration.txt index 85a418de0b4c1..47ca15b70d5ff 100644 --- a/posthog/migrations/max_migration.txt +++ b/posthog/migrations/max_migration.txt @@ -1 +1 @@ -0525_hog_function_transpiled +0526_project_field_in_taxonomy diff --git a/posthog/models/event_definition.py b/posthog/models/event_definition.py index 3f5c8d8a6f45b..bf24bd2f45b90 100644 --- a/posthog/models/event_definition.py +++ b/posthog/models/event_definition.py @@ -13,6 +13,7 @@ class EventDefinition(UUIDModel): related_name="event_definitions", related_query_name="team", ) + project = models.ForeignKey("Project", on_delete=models.CASCADE, null=True) name = models.CharField(max_length=400) created_at = models.DateTimeField(default=timezone.now, null=True) last_seen_at = models.DateTimeField(default=None, null=True) diff --git a/posthog/models/event_property.py b/posthog/models/event_property.py index a5087c5b3f667..f9a54827cddf4 100644 --- a/posthog/models/event_property.py +++ b/posthog/models/event_property.py @@ -6,6 +6,7 @@ class EventProperty(models.Model): team = models.ForeignKey(Team, on_delete=models.CASCADE) + project = models.ForeignKey("Project", on_delete=models.CASCADE, null=True) event = models.CharField(max_length=400, null=False) property = models.CharField(max_length=400, null=False) diff --git a/posthog/models/property_definition.py b/posthog/models/property_definition.py index e2e77eba2148c..1719c4007e97e 100644 --- a/posthog/models/property_definition.py +++ b/posthog/models/property_definition.py @@ -43,6 +43,7 @@ class Type(models.IntegerChoices): related_name="property_definitions", related_query_name="team", ) + project = models.ForeignKey("Project", on_delete=models.CASCADE, null=True) name = models.CharField(max_length=400) is_numerical = models.BooleanField( default=False diff --git a/rust/property-defs-rs/src/types.rs b/rust/property-defs-rs/src/types.rs index 892ddcc4b5534..e2c09c129934c 100644 --- a/rust/property-defs-rs/src/types.rs +++ b/rust/property-defs-rs/src/types.rs @@ -133,6 +133,7 @@ impl Update { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Event { pub team_id: i32, + pub project_id: i32, pub event: String, pub properties: Option, } @@ -419,14 +420,15 @@ impl EventDefinition { { sqlx::query!( r#" - INSERT INTO posthog_eventdefinition (id, name, volume_30_day, query_usage_30_day, team_id, last_seen_at, created_at) - VALUES ($1, $2, NULL, NULL, $3, $4, NOW()) ON CONFLICT + INSERT INTO posthog_eventdefinition (id, name, volume_30_day, query_usage_30_day, team_id, project_id, last_seen_at, created_at) + VALUES ($1, $2, NULL, NULL, $3, $4, $5, NOW()) ON CONFLICT ON CONSTRAINT posthog_eventdefinition_team_id_name_80fa0b87_uniq DO UPDATE SET last_seen_at = $4 "#, Uuid::now_v7(), self.name, self.team_id, + self.project_id, Utc::now() // We floor the update datetime to the nearest day for cache purposes, but can insert the exact time we see the event ).execute(executor).await.map(|_| ()) } @@ -462,8 +464,8 @@ impl PropertyDefinition { sqlx::query!( r#" - INSERT INTO posthog_propertydefinition (id, name, type, group_type_index, is_numerical, volume_30_day, query_usage_30_day, team_id, property_type) - VALUES ($1, $2, $3, $4, $5, NULL, NULL, $6, $7) + INSERT INTO posthog_propertydefinition (id, name, type, group_type_index, is_numerical, volume_30_day, query_usage_30_day, team_id, project_id, property_type) + VALUES ($1, $2, $3, $4, $5, NULL, NULL, $6, $7, $8) ON CONFLICT (team_id, name, type, coalesce(group_type_index, -1)) DO UPDATE SET property_type=EXCLUDED.property_type WHERE posthog_propertydefinition.property_type IS NULL "#, @@ -473,6 +475,7 @@ impl PropertyDefinition { group_type_index, self.is_numerical, self.team_id, + self.project_id, self.property_type.as_ref().map(|t| t.to_string()) ).execute(executor).await.map(|_| ()) } @@ -484,10 +487,11 @@ impl EventProperty { E: Executor<'c, Database = Postgres>, { sqlx::query!( - r#"INSERT INTO posthog_eventproperty (event, property, team_id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING"#, + r#"INSERT INTO posthog_eventproperty (event, property, team_id, project_id) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING"#, self.event, self.property, self.team_id + self.project_id ) .execute(executor) .await From 10a006e897cda0761816236e6abe2abf11337058 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 28 Nov 2024 22:51:20 +0100 Subject: [PATCH 2/9] Fix missing indexes and SQLX cache --- .../0526_project_field_in_taxonomy.py | 85 ++++++++++++------- posthog/models/event_definition.py | 4 +- posthog/models/event_property.py | 4 + posthog/models/property_definition.py | 11 +++ ...b4dcdf2e218e48e0fd8a247e1b7ae0f04aee3.json | 21 +++++ ...88a75efdb8abe195339e0a1676ebed6fc61ef.json | 17 ++++ ...52ea79485e403b9e17b6c37259ea0612065ee.json | 23 +++++ rust/property-defs-rs/src/types.rs | 12 ++- 8 files changed, 142 insertions(+), 35 deletions(-) create mode 100644 rust/property-defs-rs/.sqlx/query-04abdef9c07ae1a30bb6f22abcfb4dcdf2e218e48e0fd8a247e1b7ae0f04aee3.json create mode 100644 rust/property-defs-rs/.sqlx/query-9e0e25b9966a23792427c27a80888a75efdb8abe195339e0a1676ebed6fc61ef.json create mode 100644 rust/property-defs-rs/.sqlx/query-c6ff00fcbbc77c8f5c1b3fe2f3352ea79485e403b9e17b6c37259ea0612065ee.json diff --git a/posthog/migrations/0526_project_field_in_taxonomy.py b/posthog/migrations/0526_project_field_in_taxonomy.py index 1d0909987c1a4..3a02522de637e 100644 --- a/posthog/migrations/0526_project_field_in_taxonomy.py +++ b/posthog/migrations/0526_project_field_in_taxonomy.py @@ -2,6 +2,9 @@ from django.db import migrations, models import django.db.models.deletion +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db.models.expressions import F, OrderBy +from django.db.models.functions.comparison import Coalesce class Migration(migrations.Migration): @@ -9,7 +12,21 @@ class Migration(migrations.Migration): dependencies = [("posthog", "0525_hog_function_transpiled")] operations = [ - migrations.SeparateDatabaseAndState( + migrations.RunSQL( + """ + -- Add field project to eventdefinition + ALTER TABLE "posthog_eventdefinition" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_eventdefinit_project_id_f93fcbb0_fk_posthog_p" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED; + SET CONSTRAINTS "posthog_eventdefinit_project_id_f93fcbb0_fk_posthog_p" IMMEDIATE; + -- Add field project to eventproperty + ALTER TABLE "posthog_eventproperty" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_eventproperty_project_id_dd2337d2_fk_posthog_project_id" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED; + SET CONSTRAINTS "posthog_eventproperty_project_id_dd2337d2_fk_posthog_project_id" IMMEDIATE; + -- Add field project to propertydefinition + ALTER TABLE "posthog_propertydefinition" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_propertydefi_project_id_d3eb982d_fk_posthog_p" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED; + SET CONSTRAINTS "posthog_propertydefi_project_id_d3eb982d_fk_posthog_p" IMMEDIATE;""", + reverse_sql=""" + ALTER TABLE "posthog_eventdefinition" DROP COLUMN IF EXISTS "project_id"; + ALTER TABLE "posthog_eventproperty" DROP COLUMN IF EXISTS "project_id"; + ALTER TABLE "posthog_propertydefinition" DROP COLUMN IF EXISTS "project_id";""", state_operations=[ migrations.AddField( model_name="eventdefinition", @@ -33,34 +50,40 @@ class Migration(migrations.Migration): ), ), ], - database_operations=[ - migrations.RunSQL( - """ - -- Add field project to eventdefinition - ALTER TABLE "posthog_eventdefinition" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_eventdefinit_project_id_f93fcbb0_fk_posthog_p" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED; - SET CONSTRAINTS "posthog_eventdefinit_project_id_f93fcbb0_fk_posthog_p" IMMEDIATE; - -- Add field project to eventproperty - ALTER TABLE "posthog_eventproperty" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_eventproperty_project_id_dd2337d2_fk_posthog_project_id" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED; - SET CONSTRAINTS "posthog_eventproperty_project_id_dd2337d2_fk_posthog_project_id" IMMEDIATE; - -- Add field project to propertydefinition - ALTER TABLE "posthog_propertydefinition" ADD COLUMN "project_id" bigint NULL CONSTRAINT "posthog_propertydefi_project_id_d3eb982d_fk_posthog_p" REFERENCES "posthog_project"("id") DEFERRABLE INITIALLY DEFERRED; - SET CONSTRAINTS "posthog_propertydefi_project_id_d3eb982d_fk_posthog_p" IMMEDIATE;""", - reverse_sql=""" - ALTER TABLE "posthog_eventdefinition" DROP COLUMN IF EXISTS "project_id"; - ALTER TABLE "posthog_eventproperty" DROP COLUMN IF EXISTS "project_id"; - ALTER TABLE "posthog_propertydefinition" DROP COLUMN IF EXISTS "project_id";""", - ), - # We add CONCURRENTLY to the CREATE INDEX commands - migrations.RunSQL( - """ - CREATE INDEX CONCURRENTLY "posthog_eventdefinition_project_id_f93fcbb0" ON "posthog_eventdefinition" ("project_id"); - CREATE INDEX CONCURRENTLY "posthog_eventproperty_project_id_dd2337d2" ON "posthog_eventproperty" ("project_id"); - CREATE INDEX CONCURRENTLY "posthog_propertydefinition_project_id_d3eb982d" ON "posthog_propertydefinition" ("project_id");""", - reverse_sql=""" - DROP INDEX IF EXISTS "posthog_eventdefinition_project_id_f93fcbb0"; - DROP INDEX IF EXISTS "posthog_eventproperty_project_id_dd2337d2"; - DROP INDEX IF EXISTS "posthog_propertydefinition_project_id_d3eb982d";""", - ), - ], - ) + ), + AddIndexConcurrently( + model_name="eventdefinition", + index=models.Index(fields=["project"], name="posthog_eve_proj_id_f93fcbb0"), + ), + AddIndexConcurrently( + model_name="propertydefinition", + index=models.Index(fields=["project"], name="posthog_prop_proj_id_d3eb982d"), + ), + AddIndexConcurrently( + model_name="propertydefinition", + index=models.Index( + F("project_id"), + F("type"), + Coalesce(F("group_type_index"), -1), + OrderBy(F("query_usage_30_day"), descending=True, nulls_last=True), + OrderBy(F("name")), + name="index_property_def_query_proj", + ), + ), + migrations.AddIndex( + model_name="propertydefinition", + index=models.Index(fields=["project_id", "type", "is_numerical"], name="posthog_pro_project_3583d2_idx"), + ), + AddIndexConcurrently( + model_name="eventproperty", + index=models.Index(fields=["project"], name="posthog_eve_proj_id_dd2337d2"), + ), + AddIndexConcurrently( + model_name="eventproperty", + index=models.Index(fields=["project", "event"], name="posthog_eve_proj_id_22de03_idx"), + ), + AddIndexConcurrently( + model_name="eventproperty", + index=models.Index(fields=["project", "property"], name="posthog_eve_proj_id_26dbfb_idx"), + ), ] diff --git a/posthog/models/event_definition.py b/posthog/models/event_definition.py index bf24bd2f45b90..47a14035652ed 100644 --- a/posthog/models/event_definition.py +++ b/posthog/models/event_definition.py @@ -29,11 +29,13 @@ class EventDefinition(UUIDModel): class Meta: unique_together = ("team", "name") indexes = [ + # Index on project_id foreign key + models.Index(fields=["project"], name="posthog_eve_proj_id_f93fcbb0"), GinIndex( name="index_event_definition_name", fields=["name"], opclasses=["gin_trgm_ops"], - ) # To speed up DB-based fuzzy searching + ), # To speed up DB-based fuzzy searching ] def __str__(self) -> str: diff --git a/posthog/models/event_property.py b/posthog/models/event_property.py index f9a54827cddf4..61f5d27baace6 100644 --- a/posthog/models/event_property.py +++ b/posthog/models/event_property.py @@ -18,8 +18,12 @@ class Meta: ) ] indexes = [ + # Index on project_id foreign key + models.Index(fields=["project"], name="posthog_eve_proj_id_dd2337d2"), models.Index(fields=["team", "event"]), + models.Index(fields=["project", "event"], name="posthog_eve_proj_id_22de03_idx"), models.Index(fields=["team", "property"]), + models.Index(fields=["project", "property"], name="posthog_eve_proj_id_26dbfb_idx"), ] __repr__ = sane_repr("event", "property", "team_id") diff --git a/posthog/models/property_definition.py b/posthog/models/property_definition.py index 1719c4007e97e..08e6cd5d1bed1 100644 --- a/posthog/models/property_definition.py +++ b/posthog/models/property_definition.py @@ -70,6 +70,8 @@ class Type(models.IntegerChoices): class Meta: indexes = [ + # Index on project_id foreign key + models.Index(fields=["project"], name="posthog_prop_proj_id_d3eb982d"), # This indexes the query in api/property_definition.py # :KLUDGE: django ORM typing is off here models.Index( @@ -80,9 +82,18 @@ class Meta: F("name").asc(), name="index_property_def_query", ), + models.Index( + F("project_id"), + F("type"), + Coalesce(F("group_type_index"), -1), + F("query_usage_30_day").desc(nulls_last=True), + F("name").asc(), + name="index_property_def_query_proj", + ), # creates an index pganalyze identified as missing # https://app.pganalyze.com/servers/i35ydkosi5cy5n7tly45vkjcqa/checks/index_advisor/missing_index/15282978 models.Index(fields=["team_id", "type", "is_numerical"]), + models.Index(fields=["project_id", "type", "is_numerical"], name="posthog_pro_project_3583d2_idx"), GinIndex( name="index_property_definition_name", fields=["name"], diff --git a/rust/property-defs-rs/.sqlx/query-04abdef9c07ae1a30bb6f22abcfb4dcdf2e218e48e0fd8a247e1b7ae0f04aee3.json b/rust/property-defs-rs/.sqlx/query-04abdef9c07ae1a30bb6f22abcfb4dcdf2e218e48e0fd8a247e1b7ae0f04aee3.json new file mode 100644 index 0000000000000..5c8b96e695c28 --- /dev/null +++ b/rust/property-defs-rs/.sqlx/query-04abdef9c07ae1a30bb6f22abcfb4dcdf2e218e48e0fd8a247e1b7ae0f04aee3.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO posthog_propertydefinition (id, name, type, group_type_index, is_numerical, volume_30_day, query_usage_30_day, team_id, project_id, property_type)\n VALUES ($1, $2, $3, $4, $5, NULL, NULL, $6, $7, $8)\n ON CONFLICT (team_id, name, type, coalesce(group_type_index, -1))\n DO UPDATE SET property_type=EXCLUDED.property_type WHERE posthog_propertydefinition.property_type IS NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Int2", + "Int2", + "Bool", + "Int4", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "04abdef9c07ae1a30bb6f22abcfb4dcdf2e218e48e0fd8a247e1b7ae0f04aee3" +} diff --git a/rust/property-defs-rs/.sqlx/query-9e0e25b9966a23792427c27a80888a75efdb8abe195339e0a1676ebed6fc61ef.json b/rust/property-defs-rs/.sqlx/query-9e0e25b9966a23792427c27a80888a75efdb8abe195339e0a1676ebed6fc61ef.json new file mode 100644 index 0000000000000..f2582dca5c9b4 --- /dev/null +++ b/rust/property-defs-rs/.sqlx/query-9e0e25b9966a23792427c27a80888a75efdb8abe195339e0a1676ebed6fc61ef.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO posthog_eventproperty (event, property, team_id, project_id) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int4", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "9e0e25b9966a23792427c27a80888a75efdb8abe195339e0a1676ebed6fc61ef" +} diff --git a/rust/property-defs-rs/.sqlx/query-c6ff00fcbbc77c8f5c1b3fe2f3352ea79485e403b9e17b6c37259ea0612065ee.json b/rust/property-defs-rs/.sqlx/query-c6ff00fcbbc77c8f5c1b3fe2f3352ea79485e403b9e17b6c37259ea0612065ee.json new file mode 100644 index 0000000000000..608008e09adb5 --- /dev/null +++ b/rust/property-defs-rs/.sqlx/query-c6ff00fcbbc77c8f5c1b3fe2f3352ea79485e403b9e17b6c37259ea0612065ee.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT group_type_index FROM posthog_grouptypemapping WHERE group_type = $1 AND team_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "group_type_index", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c6ff00fcbbc77c8f5c1b3fe2f3352ea79485e403b9e17b6c37259ea0612065ee" +} diff --git a/rust/property-defs-rs/src/types.rs b/rust/property-defs-rs/src/types.rs index e2c09c129934c..e8bac46a0f50c 100644 --- a/rust/property-defs-rs/src/types.rs +++ b/rust/property-defs-rs/src/types.rs @@ -84,6 +84,7 @@ impl GroupType { #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] pub struct PropertyDefinition { pub team_id: i32, + pub project_id: i32, pub name: String, pub is_numerical: bool, pub property_type: Option, @@ -98,6 +99,7 @@ pub struct PropertyDefinition { pub struct EventDefinition { pub name: String, pub team_id: i32, + pub project_id: i32, pub last_seen_at: DateTime, // Always floored to our update rate for last_seen, so this Eq derive is safe for deduping } @@ -105,6 +107,7 @@ pub struct EventDefinition { #[derive(Clone, Debug, Hash, Eq, PartialEq, PartialOrd, Ord)] pub struct EventProperty { pub team_id: i32, + pub project_id: i32, pub event: String, pub property: String, } @@ -143,6 +146,7 @@ impl From<&Event> for EventDefinition { EventDefinition { name: sanitize_event_name(&event.event), team_id: event.team_id, + project_id: event.project_id, // We round last seen to the nearest hour. Unwrap is safe here because we // the duration is positive, non-zero, and smaller than time since epoch. We use this // in the hash value, so updates which would modify this in the DB are issued even @@ -262,6 +266,7 @@ impl Event { updates.push(Update::EventProperty(EventProperty { team_id: self.team_id, + project_id: self.project_id, event: self.event.clone(), property: key.clone(), })); @@ -271,6 +276,7 @@ impl Event { let def = PropertyDefinition { team_id: self.team_id, + project_id: self.project_id, name: key.clone(), is_numerical, property_type, @@ -475,7 +481,7 @@ impl PropertyDefinition { group_type_index, self.is_numerical, self.team_id, - self.project_id, + self.project_id as i64, self.property_type.as_ref().map(|t| t.to_string()) ).execute(executor).await.map(|_| ()) } @@ -490,8 +496,8 @@ impl EventProperty { r#"INSERT INTO posthog_eventproperty (event, property, team_id, project_id) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING"#, self.event, self.property, - self.team_id - self.project_id + self.team_id, + self.project_id as i64 ) .execute(executor) .await From 7ec955b7d3a2b422d122d6f9b1c8a397c4fafa4a Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2024 22:20:17 +0000 Subject: [PATCH 3/9] Update query snapshots --- posthog/api/test/__snapshots__/test_api_docs.ambr | 2 -- 1 file changed, 2 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_api_docs.ambr b/posthog/api/test/__snapshots__/test_api_docs.ambr index 42c03569e8f25..8d84cb9579b88 100644 --- a/posthog/api/test/__snapshots__/test_api_docs.ambr +++ b/posthog/api/test/__snapshots__/test_api_docs.ambr @@ -34,7 +34,6 @@ '/home/runner/work/posthog/posthog/posthog/api/event.py: Warning [EventViewSet]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/event.py: Warning [EventViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', "/home/runner/work/posthog/posthog/posthog/api/event_definition.py: Error [EventDefinitionViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AnonymousUser' object has no attribute 'organization')", - '/home/runner/work/posthog/posthog/posthog/api/event_definition.py: Warning [EventDefinitionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.event_definition.EventDefinition" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet > ExportedAssetSerializer]: unable to resolve type hint for function "filename". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet > ExportedAssetSerializer]: unable to resolve type hint for function "has_content". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.exported_asset.ExportedAsset" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', @@ -59,7 +58,6 @@ '/home/runner/work/posthog/posthog/posthog/api/project.py: Warning [ProjectViewSet > ProjectBackwardCompatSerializer]: could not resolve field on model with path "person_on_events_querying_enabled". This is likely a custom field that does some unknown magic. Maybe consider annotating the field/property? Defaulting to "string". (Exception: Project has no field named \'person_on_events_querying_enabled\')', '/home/runner/work/posthog/posthog/posthog/api/project.py: Warning [ProjectViewSet > ProjectBackwardCompatSerializer]: unable to resolve type hint for function "get_product_intents". Consider using a type hint or @extend_schema_field. Defaulting to string.', "/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Error [PropertyDefinitionViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AnonymousUser' object has no attribute 'organization')", - '/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Warning [PropertyDefinitionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.property_definition.PropertyDefinition" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/proxy_record.py: Warning [ProxyRecordViewset]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/proxy_record.py: Warning [ProxyRecordViewset]: could not derive type of path parameter "organization_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/query.py: Error [QueryViewSet]: unable to guess serializer. This is graceful fallback handling for APIViews. Consider using GenericAPIView as view base class, if view is under your control. Either way you may want to add a serializer_class (or method). Ignoring view for now.', From de43aaf09c7b36f35fa90974c19f767aafed42fb Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 28 Nov 2024 23:28:39 +0100 Subject: [PATCH 4/9] Update SQLX cache more --- ...4073799ecba84594ca04cfb24481cffbf6f6ca.json | 18 ++++++++++++++++++ rust/property-defs-rs/src/types.rs | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 rust/property-defs-rs/.sqlx/query-2b9a8c4b8d323e1673d805125b4073799ecba84594ca04cfb24481cffbf6f6ca.json diff --git a/rust/property-defs-rs/.sqlx/query-2b9a8c4b8d323e1673d805125b4073799ecba84594ca04cfb24481cffbf6f6ca.json b/rust/property-defs-rs/.sqlx/query-2b9a8c4b8d323e1673d805125b4073799ecba84594ca04cfb24481cffbf6f6ca.json new file mode 100644 index 0000000000000..785a13a6d1ce7 --- /dev/null +++ b/rust/property-defs-rs/.sqlx/query-2b9a8c4b8d323e1673d805125b4073799ecba84594ca04cfb24481cffbf6f6ca.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO posthog_eventdefinition (id, name, volume_30_day, query_usage_30_day, team_id, project_id, last_seen_at, created_at)\n VALUES ($1, $2, NULL, NULL, $3, $4, $5, NOW()) ON CONFLICT\n ON CONSTRAINT posthog_eventdefinition_team_id_name_80fa0b87_uniq\n DO UPDATE SET last_seen_at = $5\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Int4", + "Int8", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "2b9a8c4b8d323e1673d805125b4073799ecba84594ca04cfb24481cffbf6f6ca" +} diff --git a/rust/property-defs-rs/src/types.rs b/rust/property-defs-rs/src/types.rs index e8bac46a0f50c..10279e09e5839 100644 --- a/rust/property-defs-rs/src/types.rs +++ b/rust/property-defs-rs/src/types.rs @@ -429,12 +429,12 @@ impl EventDefinition { INSERT INTO posthog_eventdefinition (id, name, volume_30_day, query_usage_30_day, team_id, project_id, last_seen_at, created_at) VALUES ($1, $2, NULL, NULL, $3, $4, $5, NOW()) ON CONFLICT ON CONSTRAINT posthog_eventdefinition_team_id_name_80fa0b87_uniq - DO UPDATE SET last_seen_at = $4 + DO UPDATE SET last_seen_at = $5 "#, Uuid::now_v7(), self.name, self.team_id, - self.project_id, + self.project_id as i64, Utc::now() // We floor the update datetime to the nearest day for cache purposes, but can insert the exact time we see the event ).execute(executor).await.map(|_| ()) } From 92de540b4f62b5ce33480e9f3382bff06a0fc27f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 29 Nov 2024 00:07:11 +0100 Subject: [PATCH 5/9] Fix non-concurrent index creation --- posthog/migrations/0526_project_field_in_taxonomy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/migrations/0526_project_field_in_taxonomy.py b/posthog/migrations/0526_project_field_in_taxonomy.py index 3a02522de637e..fa907f1b7f7d6 100644 --- a/posthog/migrations/0526_project_field_in_taxonomy.py +++ b/posthog/migrations/0526_project_field_in_taxonomy.py @@ -70,7 +70,7 @@ class Migration(migrations.Migration): name="index_property_def_query_proj", ), ), - migrations.AddIndex( + AddIndexConcurrently( model_name="propertydefinition", index=models.Index(fields=["project_id", "type", "is_numerical"], name="posthog_pro_project_3583d2_idx"), ), From 52d27502b2b764e673bfe1d09461481bbb5d75f8 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 29 Nov 2024 00:08:48 +0100 Subject: [PATCH 6/9] Update updates.rs --- rust/property-defs-rs/tests/updates.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/property-defs-rs/tests/updates.rs b/rust/property-defs-rs/tests/updates.rs index 773245a24dedd..8e78eeed00368 100644 --- a/rust/property-defs-rs/tests/updates.rs +++ b/rust/property-defs-rs/tests/updates.rs @@ -18,6 +18,7 @@ async fn test_updates(db: PgPool) { let event_src = json!({ "team_id": 1, + "project_id": 1, "event": "update", "properties": properties }); From 1c76d73ed70a2db309295053451b5e424d7155ac Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 29 Nov 2024 00:17:41 +0100 Subject: [PATCH 7/9] Update `test_my_flags_is_not_nplus1` --- posthog/api/test/test_feature_flag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/api/test/test_feature_flag.py b/posthog/api/test/test_feature_flag.py index 86cee8950dfdf..4fd4183c1ac38 100644 --- a/posthog/api/test/test_feature_flag.py +++ b/posthog/api/test/test_feature_flag.py @@ -1259,7 +1259,7 @@ def test_my_flags_is_not_nplus1(self) -> None: format="json", ).json() - with self.assertNumQueries(FuzzyInt(7, 8)): + with self.assertNumQueries(FuzzyInt(8, 9)): response = self.client.get(f"/api/projects/{self.team.id}/feature_flags/my_flags") self.assertEqual(response.status_code, status.HTTP_200_OK) From 38a1242a70aa0f49f2888a56476a91f599176658 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 29 Nov 2024 17:28:43 +0100 Subject: [PATCH 8/9] Update 20240830124836_setup_propdefs_tables.sql --- .../20240830124836_setup_propdefs_tables.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rust/property-defs-rs/tests/test_migrations/20240830124836_setup_propdefs_tables.sql b/rust/property-defs-rs/tests/test_migrations/20240830124836_setup_propdefs_tables.sql index ec83e55a73dae..30995aca47fc5 100644 --- a/rust/property-defs-rs/tests/test_migrations/20240830124836_setup_propdefs_tables.sql +++ b/rust/property-defs-rs/tests/test_migrations/20240830124836_setup_propdefs_tables.sql @@ -9,6 +9,7 @@ CREATE TABLE IF NOT EXISTS posthog_eventdefinition ( volume_30_day INTEGER, query_usage_30_day INTEGER, team_id INTEGER NOT NULL, + project_id INTEGER NULL, last_seen_at TIMESTAMPTZ NOT NULL, created_at TIMESTAMPTZ NOT NULL, CONSTRAINT posthog_eventdefinition_team_id_name_80fa0b87_uniq UNIQUE (team_id, name) @@ -24,6 +25,7 @@ CREATE TABLE IF NOT EXISTS posthog_propertydefinition ( property_type_format VARCHAR(50), volume_30_day INTEGER, team_id INTEGER NOT NULL, + project_id INTEGER NULL, group_type_index SMALLINT, type SMALLINT NOT NULL DEFAULT 1 ); @@ -35,7 +37,8 @@ CREATE TABLE IF NOT EXISTS posthog_eventproperty ( id SERIAL PRIMARY KEY, event VARCHAR(400)NOT NULL, property VARCHAR(400) NOT NULL, - team_id INTEGER NOT NULL + team_id INTEGER NOT NULL, + project_id INTEGER NULL ); -CREATE UNIQUE INDEX posthog_event_property_unique_team_event_property ON posthog_eventproperty (team_id, event, property); \ No newline at end of file +CREATE UNIQUE INDEX posthog_event_property_unique_team_event_property ON posthog_eventproperty (team_id, event, property); From 606d3c0c18bd20d5047521df45ba315429f17d79 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 4 Dec 2024 14:45:07 +0100 Subject: [PATCH 9/9] Ensure `project_id` is `i64` --- rust/common/types/src/event.rs | 2 +- rust/property-defs-rs/src/types.rs | 14 +++++++------- .../20240830124836_setup_propdefs_tables.sql | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/rust/common/types/src/event.rs b/rust/common/types/src/event.rs index 5ce5efb62e8cf..309f86c35543c 100644 --- a/rust/common/types/src/event.rs +++ b/rust/common/types/src/event.rs @@ -39,7 +39,7 @@ pub enum PersonMode { pub struct ClickHouseEvent { pub uuid: Uuid, pub team_id: i32, - pub project_id: i32, + pub project_id: i64, pub event: String, pub distinct_id: String, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/rust/property-defs-rs/src/types.rs b/rust/property-defs-rs/src/types.rs index 10279e09e5839..d437c62849f0e 100644 --- a/rust/property-defs-rs/src/types.rs +++ b/rust/property-defs-rs/src/types.rs @@ -84,7 +84,7 @@ impl GroupType { #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] pub struct PropertyDefinition { pub team_id: i32, - pub project_id: i32, + pub project_id: i64, pub name: String, pub is_numerical: bool, pub property_type: Option, @@ -99,7 +99,7 @@ pub struct PropertyDefinition { pub struct EventDefinition { pub name: String, pub team_id: i32, - pub project_id: i32, + pub project_id: i64, pub last_seen_at: DateTime, // Always floored to our update rate for last_seen, so this Eq derive is safe for deduping } @@ -107,7 +107,7 @@ pub struct EventDefinition { #[derive(Clone, Debug, Hash, Eq, PartialEq, PartialOrd, Ord)] pub struct EventProperty { pub team_id: i32, - pub project_id: i32, + pub project_id: i64, pub event: String, pub property: String, } @@ -136,7 +136,7 @@ impl Update { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Event { pub team_id: i32, - pub project_id: i32, + pub project_id: i64, pub event: String, pub properties: Option, } @@ -434,7 +434,7 @@ impl EventDefinition { Uuid::now_v7(), self.name, self.team_id, - self.project_id as i64, + self.project_id, Utc::now() // We floor the update datetime to the nearest day for cache purposes, but can insert the exact time we see the event ).execute(executor).await.map(|_| ()) } @@ -481,7 +481,7 @@ impl PropertyDefinition { group_type_index, self.is_numerical, self.team_id, - self.project_id as i64, + self.project_id, self.property_type.as_ref().map(|t| t.to_string()) ).execute(executor).await.map(|_| ()) } @@ -497,7 +497,7 @@ impl EventProperty { self.event, self.property, self.team_id, - self.project_id as i64 + self.project_id ) .execute(executor) .await diff --git a/rust/property-defs-rs/tests/test_migrations/20240830124836_setup_propdefs_tables.sql b/rust/property-defs-rs/tests/test_migrations/20240830124836_setup_propdefs_tables.sql index 30995aca47fc5..69b8e56b9e400 100644 --- a/rust/property-defs-rs/tests/test_migrations/20240830124836_setup_propdefs_tables.sql +++ b/rust/property-defs-rs/tests/test_migrations/20240830124836_setup_propdefs_tables.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS posthog_eventdefinition ( volume_30_day INTEGER, query_usage_30_day INTEGER, team_id INTEGER NOT NULL, - project_id INTEGER NULL, + project_id BIGINT NULL, last_seen_at TIMESTAMPTZ NOT NULL, created_at TIMESTAMPTZ NOT NULL, CONSTRAINT posthog_eventdefinition_team_id_name_80fa0b87_uniq UNIQUE (team_id, name) @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS posthog_propertydefinition ( property_type_format VARCHAR(50), volume_30_day INTEGER, team_id INTEGER NOT NULL, - project_id INTEGER NULL, + project_id BIGINT NULL, group_type_index SMALLINT, type SMALLINT NOT NULL DEFAULT 1 ); @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS posthog_eventproperty ( event VARCHAR(400)NOT NULL, property VARCHAR(400) NOT NULL, team_id INTEGER NOT NULL, - project_id INTEGER NULL + project_id BIGINT NULL ); CREATE UNIQUE INDEX posthog_event_property_unique_team_event_property ON posthog_eventproperty (team_id, event, property);