Skip to content

Commit

Permalink
fix: Swap to raw sql for person deletes command (#26863)
Browse files Browse the repository at this point in the history 
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@benjackwhite @github-actions
benjackwhite and github-actions[bot] authored Dec 12, 2024
1 parent db25fd8 commit aa2a026
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 18 deletions.
14 changes: 14 additions & 0 deletions posthog/api/test/__snapshots__/test_decide.ambr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3890,6 +3890,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id",
"posthog_team"."id",
Expand Down Expand Up @@ -4260,6 +4261,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -4693,6 +4695,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -4986,6 +4989,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -5171,6 +5175,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id",
"posthog_team"."id",
Expand Down Expand Up @@ -5602,6 +5607,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -5834,6 +5840,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -6165,6 +6172,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -6344,6 +6352,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -6475,6 +6484,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id",
"posthog_team"."id",
Expand Down Expand Up @@ -6906,6 +6916,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -7138,6 +7149,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -7465,6 +7477,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down Expand Up @@ -7640,6 +7653,7 @@
"posthog_hogfunction"."inputs",
"posthog_hogfunction"."encrypted_inputs",
"posthog_hogfunction"."filters",
"posthog_hogfunction"."mappings",
"posthog_hogfunction"."masking",
"posthog_hogfunction"."template_id"
FROM "posthog_hogfunction"
Expand Down
69 changes: 51 additions & 18 deletions posthog/management/commands/delete_persons.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import logging

from django.db import connection
import structlog
from django.core.management.base import BaseCommand

from posthog.models.person.person import Person

logger = structlog.get_logger(__name__)
logger.setLevel(logging.INFO)
Expand Down Expand Up @@ -40,27 +40,54 @@ def run(options, sync: bool = False):
logger.info(f"-> Team ID: {team_id}")
if person_ids:
logger.info(f"-> Person IDs: {person_ids}")
logger.info(f"-> Limit: {limit}")

list_query = Person.objects.filter(team_id=team_id)

if person_ids:
list_query = list_query.filter(id__in=person_ids)

list_query = list_query.order_by("id")[:limit]
logger.info(f"-> Limit: {limit} ")

select_query = f"""
SELECT id
FROM posthog_person
WHERE team_id=%(team_id)s {f"AND id IN ({person_ids})" if person_ids else ""}
ORDER BY id ASC
LIMIT %(limit)s
"""

delete_query_person_distinct_ids = f"""
WITH to_delete AS ({select_query})
DELETE FROM posthog_persondistinctid
WHERE team_id = %(team_id)s AND person_id IN (SELECT id FROM to_delete);
"""

delete_query_person_override = f"""
WITH to_delete AS ({select_query})
DELETE FROM posthog_personoverride
WHERE team_id = %(team_id)s AND (old_person_id IN (SELECT id FROM to_delete) OR override_person_id IN (SELECT id FROM to_delete));
"""

delete_query_person = f"""
WITH to_delete AS ({select_query})
DELETE FROM posthog_person
WHERE team_id = %(team_id)s AND id IN (SELECT id FROM to_delete);
"""

with connection.cursor() as cursor:
prepared_person_distinct_ids_query = cursor.mogrify(
delete_query_person_distinct_ids, {"team_id": team_id, "limit": limit, "person_ids": person_ids}
)
prepared_person_override_query = cursor.mogrify(
delete_query_person_override, {"team_id": team_id, "limit": limit, "person_ids": person_ids}
)
prepared_person_query = cursor.mogrify(
delete_query_person, {"team_id": team_id, "limit": limit, "person_ids": person_ids}
)

num_to_delete = list_query.count()
logger.info(f"Delete query to run:")
logger.info(prepared_person_distinct_ids_query)
logger.info(prepared_person_override_query)
logger.info(prepared_person_query)

if not live_run:
logger.info(f"Dry run. Would have deleted {num_to_delete} people.")
logger.info("Set --live-run to actually delete.")
return exit(0)

if num_to_delete == 0:
logger.info("No people to delete")
logger.info(f"Dry run. Set --live-run to actually delete.")
return exit(0)

logger.info(f"Will run the deletion for {num_to_delete} people.")
confirm = input("Type 'delete' to confirm: ")

if confirm != "delete":
Expand All @@ -70,6 +97,12 @@ def run(options, sync: bool = False):
logger.info(f"Executing delete query...")

# distinct_ids are deleted by cascade
Person.objects.filter(team_id=team_id, id__in=list_query.values_list("id", flat=True)).delete()
with connection.cursor() as cursor:
cursor.execute(delete_query_person_distinct_ids, {"team_id": team_id, "limit": limit, "person_ids": person_ids})
logger.info(f"Deleted {cursor.rowcount} distinct_ids")
cursor.execute(delete_query_person_override, {"team_id": team_id, "limit": limit, "person_ids": person_ids})
logger.info(f"Deleted {cursor.rowcount} person overrides")
cursor.execute(delete_query_person, {"team_id": team_id, "limit": limit, "person_ids": person_ids})
logger.info(f"Deleted {cursor.rowcount} persons")

logger.info("Done")

0 comments on commit aa2a026

Please sign in to comment.