Skip to content

Commit

Permalink
fix: Wait for 30s on 5 minute batch exports (#26517)
Browse files Browse the repository at this point in the history 
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@tomasfarias @github-actions
tomasfarias and github-actions[bot] authored Nov 28, 2024
1 parent c94795b commit 1cc905e
Show file tree
Hide file tree
Showing 4 changed files with 45 additions and 25 deletions.
2 changes: 1 addition & 1 deletion posthog/batch_exports/service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -687,7 +687,7 @@ def sync_batch_export(batch_export: BatchExport, created: bool):
start_at=batch_export.start_at,
end_at=batch_export.end_at,
intervals=[ScheduleIntervalSpec(every=batch_export.interval_time_delta)],
jitter=(batch_export.interval_time_delta / 6),
jitter=min(dt.timedelta(minutes=1), (batch_export.interval_time_delta / 6)),
time_zone_name=batch_export.team.timezone,
),
state=state,
Expand Down
48 changes: 24 additions & 24 deletions posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -640,12 +640,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '446'
AND "ee_accesscontrol"."resource_id" = '437'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '446'
AND "ee_accesscontrol"."resource_id" = '437'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -1688,12 +1688,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -2441,12 +2441,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -3129,12 +3129,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -3881,12 +3881,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -4597,12 +4597,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -5395,12 +5395,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -5659,12 +5659,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -6091,12 +6091,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -6556,12 +6556,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -7248,12 +7248,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down Expand Up @@ -7997,12 +7997,12 @@
LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id")
WHERE (("ee_accesscontrol"."organization_member_id" IS NULL
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("posthog_organizationmembership"."user_id" = 99999
AND "ee_accesscontrol"."resource" = 'project'
AND "ee_accesscontrol"."resource_id" = '453'
AND "ee_accesscontrol"."resource_id" = '444'
AND "ee_accesscontrol"."role_id" IS NULL
AND "ee_accesscontrol"."team_id" = 99999)
OR ("ee_accesscontrol"."organization_member_id" IS NULL
Expand Down
16 changes: 16 additions & 0 deletions posthog/temporal/batch_exports/batch_exports.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1153,3 +1153,19 @@ async def execute_batch_export_insert_activity(
non_retryable_error_types=["NotNullViolation", "IntegrityError"],
),
)


async def wait_for_delta_past_data_interval_end(
data_interval_end: dt.datetime, delta: dt.timedelta = dt.timedelta(seconds=30)
) -> None:
"""Wait for some time after `data_interval_end` before querying ClickHouse."""
if settings.TEST:
return

target = data_interval_end.astimezone(dt.UTC)
now = dt.datetime.now(dt.UTC)

while target + delta < now:
remaining = (target + delta) - now
# Sleep between 1-10 seconds, there shouldn't ever be the need to wait too long.
await asyncio.sleep(min(max(remaining.total_seconds(), 1), 10))
4 changes: 4 additions & 0 deletions posthog/temporal/batch_exports/s3_batch_export.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
get_data_interval,
iter_model_records,
start_batch_export_run,
wait_for_delta_past_data_interval_end,
)
from posthog.temporal.batch_exports.metrics import (
get_bytes_exported_metric,
Expand Down Expand Up @@ -568,6 +569,9 @@ async def insert_into_s3_activity(inputs: S3InsertInputs) -> RecordsCompleted:
set_status_to_running_task(run_id=inputs.run_id, logger=logger),
get_client(team_id=inputs.team_id, clickhouse_url=clickhouse_url) as client,
):
if is_5_min_batch_export:
await wait_for_delta_past_data_interval_end(end_at)

if not await client.is_alive():
raise ConnectionError("Cannot establish connection to ClickHouse")

Expand Down

0 comments on commit 1cc905e

Please sign in to comment.