From ebaf242618f046ef759b321215956583689c6748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Tue, 25 Jul 2023 13:05:46 +0200 Subject: [PATCH 1/4] feat: Add projection on inserted_at --- .../0048_add_inserted_at_projection.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 posthog/clickhouse/migrations/0048_add_inserted_at_projection.py diff --git a/posthog/clickhouse/migrations/0048_add_inserted_at_projection.py b/posthog/clickhouse/migrations/0048_add_inserted_at_projection.py new file mode 100644 index 0000000000000..29e7b209fcd5b --- /dev/null +++ b/posthog/clickhouse/migrations/0048_add_inserted_at_projection.py @@ -0,0 +1,21 @@ +from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions +from posthog.settings import CLICKHOUSE_CLUSTER + +ADD_PROJECTION_SQL = """ +ALTER TABLE sharded_events +ON CLUSTER '{cluster}' +ADD PROJECTION events_inserted_at_projection ( + SELECT * ORDER BY (team_id, inserted_at, event, cityHash64(distinct_id), cityHash64(uuid)) +); +""" + +MATERIALIZE_PROJECTION_SQL = """ +ALTER TABLE sharded_events +ON CLUSTER '{cluster}' +MATERIALIZE PROJECTION events_inserted_at_projection; +""" + +operations = [ + run_sql_with_exceptions(ADD_PROJECTION_SQL.format(cluster=CLICKHOUSE_CLUSTER)), + run_sql_with_exceptions(MATERIALIZE_PROJECTION_SQL.format(cluster=CLICKHOUSE_CLUSTER)), +] From fb0033390193668bd6ffdd8a8e43e75942411b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 15 Sep 2023 15:25:19 +0200 Subject: [PATCH 2/4] chore: Bump migration --- ...serted_at_projection.py => 0049_add_inserted_at_projection.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename posthog/clickhouse/migrations/{0048_add_inserted_at_projection.py => 0049_add_inserted_at_projection.py} (100%) diff --git a/posthog/clickhouse/migrations/0048_add_inserted_at_projection.py b/posthog/clickhouse/migrations/0049_add_inserted_at_projection.py similarity index 100% rename from posthog/clickhouse/migrations/0048_add_inserted_at_projection.py rename to posthog/clickhouse/migrations/0049_add_inserted_at_projection.py From 4a62a8c15d36f6ffa89cf779a95a5ebc1bc8e2b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Tue, 26 Sep 2023 15:20:31 +0200 Subject: [PATCH 3/4] refactor: Only materialize latest partition Co-authored-by: James Greenhill --- .../clickhouse/migrations/0049_add_inserted_at_projection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/posthog/clickhouse/migrations/0049_add_inserted_at_projection.py b/posthog/clickhouse/migrations/0049_add_inserted_at_projection.py index 29e7b209fcd5b..53b8945b50172 100644 --- a/posthog/clickhouse/migrations/0049_add_inserted_at_projection.py +++ b/posthog/clickhouse/migrations/0049_add_inserted_at_projection.py @@ -12,7 +12,8 @@ MATERIALIZE_PROJECTION_SQL = """ ALTER TABLE sharded_events ON CLUSTER '{cluster}' -MATERIALIZE PROJECTION events_inserted_at_projection; +MATERIALIZE PROJECTION events_inserted_at_projection +IN PARTITION '202309'; """ operations = [ From 4102d4b0193c61b12f65e6515b358f46a1c15217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 6 Oct 2023 10:38:39 +0200 Subject: [PATCH 4/4] chore: Materialize projection on 202310 --- .../clickhouse/migrations/0049_add_inserted_at_projection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/clickhouse/migrations/0049_add_inserted_at_projection.py b/posthog/clickhouse/migrations/0049_add_inserted_at_projection.py index 53b8945b50172..f802f97e53cde 100644 --- a/posthog/clickhouse/migrations/0049_add_inserted_at_projection.py +++ b/posthog/clickhouse/migrations/0049_add_inserted_at_projection.py @@ -13,7 +13,7 @@ ALTER TABLE sharded_events ON CLUSTER '{cluster}' MATERIALIZE PROJECTION events_inserted_at_projection -IN PARTITION '202309'; +IN PARTITION '202310'; """ operations = [