diff --git a/ee/api/test/test_instance_settings.py b/ee/api/test/test_instance_settings.py index 33cf5b450d6f8..de391bfc923a7 100644 --- a/ee/api/test/test_instance_settings.py +++ b/ee/api/test/test_instance_settings.py @@ -5,7 +5,7 @@ from posthog.client import sync_execute from posthog.models.instance_setting import get_instance_setting from posthog.models.performance.sql import PERFORMANCE_EVENT_DATA_TABLE -from posthog.models.session_recording_event.sql import SESSION_RECORDING_EVENTS_DATA_TABLE +from posthog.session_recordings.sql.session_recording_event_sql import SESSION_RECORDING_EVENTS_DATA_TABLE from posthog.settings.data_stores import CLICKHOUSE_DATABASE from posthog.test.base import ClickhouseTestMixin, snapshot_clickhouse_alter_queries diff --git a/ee/clickhouse/models/test/__snapshots__/test_cohort.ambr b/ee/clickhouse/models/test/__snapshots__/test_cohort.ambr index 955c3b33da9d3..d8c1a92e6e35f 100644 --- a/ee/clickhouse/models/test/__snapshots__/test_cohort.ambr +++ b/ee/clickhouse/models/test/__snapshots__/test_cohort.ambr @@ -83,7 +83,7 @@ (SELECT pdi.person_id AS person_id, countIf(timestamp > now() - INTERVAL 2 year AND timestamp < now() - AND event = '$pageview') > 0 AS performed_event_condition_17_level_level_0_level_0_level_0_0 + AND event = '$pageview') > 0 AS performed_event_condition_15_level_level_0_level_0_level_0_0 FROM events e INNER JOIN (SELECT distinct_id, @@ -113,7 +113,7 @@ HAVING max(is_deleted) = 0 AND (((((NOT has(['something1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', ''))))))))) person ON person.person_id = behavior_query.person_id WHERE 1 = 1 - AND ((((performed_event_condition_17_level_level_0_level_0_level_0_0)))) ) as person + AND ((((performed_event_condition_15_level_level_0_level_0_level_0_0)))) ) as person UNION ALL SELECT person_id, cohort_id, @@ -148,7 +148,7 @@ (SELECT pdi.person_id AS person_id, countIf(timestamp > now() - INTERVAL 2 year AND timestamp < now() - AND event = '$pageview') > 0 AS performed_event_condition_19_level_level_0_level_0_level_0_0 + AND event = '$pageview') > 0 AS performed_event_condition_17_level_level_0_level_0_level_0_0 FROM events e INNER JOIN (SELECT distinct_id, @@ -178,7 +178,7 @@ HAVING max(is_deleted) = 0 AND (((((NOT has(['something1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', ''))))))))) person ON person.person_id = behavior_query.person_id WHERE 1 = 1 - AND ((((performed_event_condition_19_level_level_0_level_0_level_0_0)))) ) )) + AND ((((performed_event_condition_17_level_level_0_level_0_level_0_0)))) ) )) ' --- # name: TestCohort.test_cohortpeople_with_not_in_cohort_operator_for_behavioural_cohorts @@ -195,7 +195,7 @@ FROM (SELECT pdi.person_id AS person_id, minIf(timestamp, event = 'signup') >= now() - INTERVAL 15 day - AND minIf(timestamp, event = 'signup') < now() as first_time_condition_20_level_level_0_level_0_0 + AND minIf(timestamp, event = 'signup') < now() as first_time_condition_18_level_level_0_level_0_0 FROM events e INNER JOIN (SELECT distinct_id, @@ -208,7 +208,7 @@ AND event IN ['signup'] GROUP BY person_id) behavior_query WHERE 1 = 1 - AND (((first_time_condition_20_level_level_0_level_0_0))) ) as person + AND (((first_time_condition_18_level_level_0_level_0_0))) ) as person UNION ALL SELECT person_id, cohort_id, @@ -237,9 +237,9 @@ (SELECT pdi.person_id AS person_id, countIf(timestamp > now() - INTERVAL 2 year AND timestamp < now() - AND event = '$pageview') > 0 AS performed_event_condition_21_level_level_0_level_0_level_0_0, + AND event = '$pageview') > 0 AS performed_event_condition_19_level_level_0_level_0_level_0_0, minIf(timestamp, event = 'signup') >= now() - INTERVAL 15 day - AND minIf(timestamp, event = 'signup') < now() as first_time_condition_21_level_level_0_level_1_level_0_level_0_level_0_0 + AND minIf(timestamp, event = 'signup') < now() as first_time_condition_19_level_level_0_level_1_level_0_level_0_level_0_0 FROM events e INNER JOIN (SELECT distinct_id, @@ -252,8 +252,8 @@ AND event IN ['$pageview', 'signup'] GROUP BY person_id) behavior_query WHERE 1 = 1 - AND ((((performed_event_condition_21_level_level_0_level_0_level_0_0)) - AND ((((NOT first_time_condition_21_level_level_0_level_1_level_0_level_0_level_0_0)))))) ) as person + AND ((((performed_event_condition_19_level_level_0_level_0_level_0_0)) + AND ((((NOT first_time_condition_19_level_level_0_level_1_level_0_level_0_level_0_0)))))) ) as person UNION ALL SELECT person_id, cohort_id, diff --git a/ee/clickhouse/models/test/__snapshots__/test_property.ambr b/ee/clickhouse/models/test/__snapshots__/test_property.ambr index b3f6f049cf619..d27396834cf99 100644 --- a/ee/clickhouse/models/test/__snapshots__/test_property.ambr +++ b/ee/clickhouse/models/test/__snapshots__/test_property.ambr @@ -146,7 +146,7 @@ )) ', { - 'global_cohort_id_0': 1, + 'global_cohort_id_0': 47, 'global_version_0': None, }, ) diff --git a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr index f312dde127a84..9f9e01f13028a 100644 --- a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr +++ b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr @@ -1,6 +1,6 @@ # name: ClickhouseTestExperimentSecondaryResults.test_basic_secondary_metric_results ' - /* user_id:138 celery:posthog.celery.sync_insight_caching_state */ + /* user_id:128 celery:posthog.celery.sync_insight_caching_state */ SELECT team_id, date_diff('second', max(timestamp), now()) AS age FROM events diff --git a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr index be61b4ccc3d33..d185a7a063790 100644 --- a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr +++ b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr @@ -1304,50 +1304,151 @@ --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone ' - /* user_id:1 celery:posthog.celery.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + /* user_id:0 request:_snapshot_ */ + SELECT groupArray(value) + FROM + (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2020-01-01 02:10:00', 'US/Pacific') + AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2020-01-06 07:00:00', 'US/Pacific') + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0) ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone.1 ' - /* celery:posthog.celery.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + /* user_id:0 request:_snapshot_ */ + SELECT groupArray(day_start) as date, + groupArray(count) AS total, + breakdown_value + FROM + (SELECT SUM(total) as count, + day_start, + breakdown_value + FROM + (SELECT * + FROM + (SELECT toUInt16(0) AS total, + ticks.day_start as day_start, + breakdown_value + FROM + (SELECT toStartOfDay(toDateTime('2020-01-06 07:00:00', 'US/Pacific')) - toIntervalDay(number) as day_start + FROM numbers(6) + UNION ALL SELECT toStartOfDay(toDateTime('2020-01-01 02:10:00', 'US/Pacific')) as day_start) as ticks + CROSS JOIN + (SELECT breakdown_value + FROM + (SELECT ['test', 'control'] as breakdown_value) ARRAY + JOIN breakdown_value) as sec + ORDER BY breakdown_value, + day_start + UNION ALL SELECT count(*) as total, + toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'US/Pacific')) as day_start, + replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') as breakdown_value + FROM events e + WHERE e.team_id = 2 + AND event = '$pageview' + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) + AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2020-01-01 02:10:00', 'US/Pacific') + AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2020-01-06 07:00:00', 'US/Pacific') + AND replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') in (['test', 'control']) + GROUP BY day_start, + breakdown_value)) + GROUP BY day_start, + breakdown_value + ORDER BY breakdown_value, + day_start) + GROUP BY breakdown_value + ORDER BY breakdown_value ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone.2 ' - /* celery:posthog.celery.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + /* user_id:0 request:_snapshot_ */ + SELECT groupArray(value) + FROM + (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$feature_flag_called' + AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2020-01-01 02:10:00', 'US/Pacific') + AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2020-01-06 07:00:00', 'US/Pacific') + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) + AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0) ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone.3 ' - /* celery:posthog.celery.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + /* user_id:0 request:_snapshot_ */ + SELECT groupArray(day_start) as date, + groupArray(count) AS total, + breakdown_value + FROM + (SELECT SUM(total) as count, + day_start, + breakdown_value + FROM + (SELECT * + FROM + (SELECT toUInt16(0) AS total, + ticks.day_start as day_start, + breakdown_value + FROM + (SELECT toStartOfDay(toDateTime('2020-01-06 07:00:00', 'US/Pacific')) - toIntervalDay(number) as day_start + FROM numbers(6) + UNION ALL SELECT toStartOfDay(toDateTime('2020-01-01 02:10:00', 'US/Pacific')) as day_start) as ticks + CROSS JOIN + (SELECT breakdown_value + FROM + (SELECT ['control', 'test'] as breakdown_value) ARRAY + JOIN breakdown_value) as sec + ORDER BY breakdown_value, + day_start + UNION ALL SELECT count(DISTINCT person_id) as total, + toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'US/Pacific')) as day_start, + breakdown_value + FROM + (SELECT person_id, + min(timestamp) as timestamp, + breakdown_value + FROM + (SELECT pdi.person_id as person_id, timestamp, replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') as breakdown_value + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) as pdi ON events.distinct_id = pdi.distinct_id + WHERE e.team_id = 2 + AND event = '$feature_flag_called' + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) + AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) + AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2020-01-01 02:10:00', 'US/Pacific') + AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2020-01-06 07:00:00', 'US/Pacific') + AND replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') in (['control', 'test']) ) + GROUP BY person_id, + breakdown_value) AS pdi + GROUP BY day_start, + breakdown_value)) + GROUP BY day_start, + breakdown_value + ORDER BY breakdown_value, + day_start) + GROUP BY breakdown_value + ORDER BY breakdown_value ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone.4 @@ -1501,50 +1602,155 @@ --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter ' - /* user_id:3 celery:posthog.celery.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + /* user_id:0 request:_snapshot_ */ + SELECT groupArray(value) + FROM + (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND ((has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) + AND (ifNull(ilike(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'hogql'), ''), 'null'), '^"|"$', ''), 'true'), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'hogql'), ''), 'null'), '^"|"$', '')) + and isNull('true')))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0) ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter.1 ' - /* celery:posthog.celery.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + /* user_id:0 request:_snapshot_ */ + SELECT groupArray(day_start) as date, + groupArray(count) AS total, + breakdown_value + FROM + (SELECT SUM(total) as count, + day_start, + breakdown_value + FROM + (SELECT * + FROM + (SELECT toUInt16(0) AS total, + ticks.day_start as day_start, + breakdown_value + FROM + (SELECT toStartOfDay(toDateTime('2020-01-06 00:00:00', 'UTC')) - toIntervalDay(number) as day_start + FROM numbers(6) + UNION ALL SELECT toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')) as day_start) as ticks + CROSS JOIN + (SELECT breakdown_value + FROM + (SELECT ['test', 'control'] as breakdown_value) ARRAY + JOIN breakdown_value) as sec + ORDER BY breakdown_value, + day_start + UNION ALL SELECT count(*) as total, + toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, + replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') as breakdown_value + FROM events e + WHERE e.team_id = 2 + AND event = '$pageview' + AND ((has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) + AND (ifNull(ilike(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'hogql'), ''), 'null'), '^"|"$', ''), 'true'), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'hogql'), ''), 'null'), '^"|"$', '')) + and isNull('true')))) + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') in (['test', 'control']) + GROUP BY day_start, + breakdown_value)) + GROUP BY day_start, + breakdown_value + ORDER BY breakdown_value, + day_start) + GROUP BY breakdown_value + ORDER BY breakdown_value ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter.2 ' - /* celery:posthog.celery.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + /* user_id:0 request:_snapshot_ */ + SELECT groupArray(value) + FROM + (SELECT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$feature_flag_called' + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) + AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 25 + OFFSET 0) ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter.3 ' - /* celery:posthog.celery.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + /* user_id:0 request:_snapshot_ */ + SELECT groupArray(day_start) as date, + groupArray(count) AS total, + breakdown_value + FROM + (SELECT SUM(total) as count, + day_start, + breakdown_value + FROM + (SELECT * + FROM + (SELECT toUInt16(0) AS total, + ticks.day_start as day_start, + breakdown_value + FROM + (SELECT toStartOfDay(toDateTime('2020-01-06 00:00:00', 'UTC')) - toIntervalDay(number) as day_start + FROM numbers(6) + UNION ALL SELECT toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')) as day_start) as ticks + CROSS JOIN + (SELECT breakdown_value + FROM + (SELECT ['control', 'test'] as breakdown_value) ARRAY + JOIN breakdown_value) as sec + ORDER BY breakdown_value, + day_start + UNION ALL SELECT count(DISTINCT person_id) as total, + toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, + breakdown_value + FROM + (SELECT person_id, + min(timestamp) as timestamp, + breakdown_value + FROM + (SELECT pdi.person_id as person_id, timestamp, replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') as breakdown_value + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) as pdi ON events.distinct_id = pdi.distinct_id + WHERE e.team_id = 2 + AND event = '$feature_flag_called' + AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) + AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') + AND replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') in (['control', 'test']) ) + GROUP BY person_id, + breakdown_value) AS pdi + GROUP BY day_start, + breakdown_value)) + GROUP BY day_start, + breakdown_value + ORDER BY breakdown_value, + day_start) + GROUP BY breakdown_value + ORDER BY breakdown_value ' --- # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter.4 diff --git a/posthog/queries/session_recordings/__init__.py b/ee/session_recordings/__init__.py similarity index 100% rename from posthog/queries/session_recordings/__init__.py rename to ee/session_recordings/__init__.py diff --git a/ee/tasks/session_recording/persistence.py b/ee/session_recordings/persistence_tasks.py similarity index 81% rename from ee/tasks/session_recording/persistence.py rename to ee/session_recordings/persistence_tasks.py index 7ab15991d8cb6..e409d9d318df9 100644 --- a/ee/tasks/session_recording/persistence.py +++ b/ee/session_recordings/persistence_tasks.py @@ -3,9 +3,9 @@ import structlog from django.utils import timezone -from ee.models.session_recording_extensions import persist_recording +from ee.session_recordings.session_recording_extensions import persist_recording from posthog.celery import app -from posthog.models.session_recording.session_recording import SessionRecording +from posthog.session_recordings.models.session_recording import SessionRecording logger = structlog.get_logger(__name__) diff --git a/ee/models/session_recording_extensions.py b/ee/session_recordings/session_recording_extensions.py similarity index 95% rename from ee/models/session_recording_extensions.py rename to ee/session_recordings/session_recording_extensions.py index 5fc7750411a59..f5d48fb8b285f 100644 --- a/ee/models/session_recording_extensions.py +++ b/ee/session_recordings/session_recording_extensions.py @@ -11,8 +11,8 @@ from posthog import settings from posthog.event_usage import report_team_action -from posthog.models.session_recording.metadata import PersistedRecordingV1 -from posthog.models.session_recording.session_recording import SessionRecording +from posthog.session_recordings.models.metadata import PersistedRecordingV1 +from posthog.session_recordings.models.session_recording import SessionRecording from posthog.session_recordings.session_recording_helpers import compress_to_string, decompress from posthog.storage import object_storage @@ -137,8 +137,8 @@ def load_persisted_recording(recording: SessionRecording) -> Optional[PersistedR # and will not be loaded here if not recording.storage_version: try: - content = object_storage.read(recording.object_storage_path) - decompressed = json.loads(decompress(content)) + content = object_storage.read(str(recording.object_storage_path)) + decompressed = json.loads(decompress(content)) if content else None logger.info( "Persisting recording load: loaded!", recording_id=recording.session_id, diff --git a/ee/api/session_recording_playlist.py b/ee/session_recordings/session_recording_playlist.py similarity index 99% rename from ee/api/session_recording_playlist.py rename to ee/session_recordings/session_recording_playlist.py index 473576c053172..72ee7915cb111 100644 --- a/ee/api/session_recording_playlist.py +++ b/ee/session_recordings/session_recording_playlist.py @@ -12,7 +12,7 @@ from posthog.api.forbid_destroy_model import ForbidDestroyModel from posthog.api.routing import StructuredViewSetMixin -from posthog.api.session_recording import list_recordings +from posthog.session_recordings.session_recording_api import list_recordings from posthog.api.shared import UserBasicSerializer from posthog.constants import SESSION_RECORDINGS_FILTER_IDS, AvailableFeature from posthog.models import SessionRecording, SessionRecordingPlaylist, SessionRecordingPlaylistItem, Team, User diff --git a/posthog/queries/session_recordings/test/test_session_recording_list.py b/ee/session_recordings/test/__init__.py similarity index 100% rename from posthog/queries/session_recordings/test/test_session_recording_list.py rename to ee/session_recordings/test/__init__.py diff --git a/ee/models/test/test_session_recording_extensions.py b/ee/session_recordings/test/test_session_recording_extensions.py similarity index 94% rename from ee/models/test/test_session_recording_extensions.py rename to ee/session_recordings/test/test_session_recording_extensions.py index de4d83ecc004c..e201e71a02563 100644 --- a/ee/models/test/test_session_recording_extensions.py +++ b/ee/session_recordings/test/test_session_recording_extensions.py @@ -7,11 +7,11 @@ from botocore.config import Config from freezegun import freeze_time -from ee.models.session_recording_extensions import load_persisted_recording, persist_recording -from posthog.models.session_recording.session_recording import SessionRecording -from posthog.models.session_recording_playlist.session_recording_playlist import SessionRecordingPlaylist -from posthog.models.session_recording_playlist_item.session_recording_playlist_item import SessionRecordingPlaylistItem -from posthog.queries.session_recordings.test.session_replay_sql import produce_replay_summary +from ee.session_recordings.session_recording_extensions import load_persisted_recording, persist_recording +from posthog.session_recordings.models.session_recording import SessionRecording +from posthog.session_recordings.models.session_recording_playlist import SessionRecordingPlaylist +from posthog.session_recordings.models.session_recording_playlist_item import SessionRecordingPlaylistItem +from posthog.session_recordings.queries.test.session_replay_sql import produce_replay_summary from posthog.session_recordings.test.test_factory import create_session_recording_events from posthog.settings import ( OBJECT_STORAGE_ENDPOINT, @@ -200,7 +200,7 @@ def test_persists_recording_from_blob_ingested_storage(self): f"{recording.build_object_storage_path('2023-08-01')}/c", ] - @patch("ee.models.session_recording_extensions.report_team_action") + @patch("ee.session_recordings.session_recording_extensions.report_team_action") def test_persist_tracks_correct_to_posthog(self, mock_capture): two_minutes_ago = (datetime.now() - timedelta(minutes=2)).replace(tzinfo=timezone.utc) diff --git a/ee/api/test/test_session_recording_playlist.py b/ee/session_recordings/test/test_session_recording_playlist.py similarity index 98% rename from ee/api/test/test_session_recording_playlist.py rename to ee/session_recordings/test/test_session_recording_playlist.py index 677cc83edc595..ddbb4d1195bca 100644 --- a/ee/api/test/test_session_recording_playlist.py +++ b/ee/session_recordings/test/test_session_recording_playlist.py @@ -12,7 +12,7 @@ from ee.api.test.base import APILicensedTest from ee.api.test.fixtures.available_product_features import AVAILABLE_PRODUCT_FEATURES from posthog.models import SessionRecording, SessionRecordingPlaylistItem -from posthog.models.session_recording_playlist.session_recording_playlist import SessionRecordingPlaylist +from posthog.session_recordings.models.session_recording_playlist import SessionRecordingPlaylist from posthog.models.user import User from posthog.session_recordings.test.test_factory import create_session_recording_events from posthog.settings import ( @@ -209,8 +209,8 @@ def test_get_pinned_recordings_for_playlist(self): assert {x["id"] for x in result["results"]} == {session_one, session_two} assert {x["pinned_count"] for x in result["results"]} == {1, 1} - @patch("ee.models.session_recording_extensions.object_storage.list_objects") - @patch("ee.models.session_recording_extensions.object_storage.copy_objects") + @patch("ee.session_recordings.session_recording_extensions.object_storage.list_objects") + @patch("ee.session_recordings.session_recording_extensions.object_storage.copy_objects") def test_fetch_playlist_recordings(self, mock_copy_objects: MagicMock, mock_list_objects: MagicMock) -> None: # all sessions have been blob ingested and had data to copy into the LTS storage location mock_copy_objects.return_value = 1 diff --git a/ee/tasks/__init__.py b/ee/tasks/__init__.py index e57f3c74cb8f9..dd549cd0c2789 100644 --- a/ee/tasks/__init__.py +++ b/ee/tasks/__init__.py @@ -1,4 +1,4 @@ -from .session_recording.persistence import persist_finished_recordings, persist_single_recording +from ee.session_recordings.persistence_tasks import persist_finished_recordings, persist_single_recording from .subscriptions import deliver_subscription_report, handle_subscription_value_change, schedule_all_subscriptions # As our EE tasks are not included at startup for Celery, we need to ensure they are declared here so that they are imported by posthog/settings/celery.py diff --git a/ee/urls.py b/ee/urls.py index 8179398ec1066..02f6028a1adcd 100644 --- a/ee/urls.py +++ b/ee/urls.py @@ -21,9 +21,9 @@ organization_resource_access, role, sentry_stats, - session_recording_playlist, subscription, ) +from .session_recordings import session_recording_playlist def extend_api_router( diff --git a/posthog/api/__init__.py b/posthog/api/__init__.py index 263b0b5f154aa..dc84bf86b158f 100644 --- a/posthog/api/__init__.py +++ b/posthog/api/__init__.py @@ -4,7 +4,6 @@ from posthog.batch_exports import http as batch_exports from posthog.settings import EE_AVAILABLE from posthog.warehouse.api import saved_query, table, view_link - from . import ( activity_log, annotation, @@ -41,6 +40,7 @@ ) from .dashboards import dashboard, dashboard_templates from .data_management import DataManagementViewSet +from ..session_recordings.session_recording_api import SessionRecordingViewSet @decorators.api_view(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"]) @@ -212,7 +212,6 @@ def api_not_found(request): from posthog.api.event import EventViewSet, LegacyEventViewSet # noqa: E402 from posthog.api.insight import InsightViewSet # noqa: E402 from posthog.api.person import LegacyPersonViewSet, PersonViewSet # noqa: E402 -from posthog.api.session_recording import SessionRecordingViewSet # noqa: E402 # Legacy endpoints CH (to be removed eventually) router.register(r"cohort", LegacyCohortViewSet, basename="cohort") diff --git a/posthog/api/instance_settings.py b/posthog/api/instance_settings.py index 9c76eef4f3d46..ab8b0fd45e8e3 100644 --- a/posthog/api/instance_settings.py +++ b/posthog/api/instance_settings.py @@ -83,7 +83,7 @@ def update(self, instance: InstanceSettingHelper, validated_data: Dict[str, Any] # TODO: Move to top-level imports once CH is moved out of `ee` from posthog.client import sync_execute - from posthog.models.session_recording_event.sql import UPDATE_RECORDINGS_TABLE_TTL_SQL + from posthog.session_recordings.sql.session_recording_event_sql import UPDATE_RECORDINGS_TABLE_TTL_SQL sync_execute(UPDATE_RECORDINGS_TABLE_TTL_SQL(), {"weeks": new_value_parsed}) diff --git a/posthog/api/sharing.py b/posthog/api/sharing.py index 072e93d97e9b2..ccbc8b5f68794 100644 --- a/posthog/api/sharing.py +++ b/posthog/api/sharing.py @@ -15,15 +15,15 @@ from posthog.api.exports import ExportedAssetSerializer from posthog.api.insight import InsightSerializer from posthog.api.routing import StructuredViewSetMixin -from posthog.api.session_recording import SessionRecordingSerializer from posthog.models import SharingConfiguration, Team from posthog.models.activity_logging.activity_log import log_activity, Detail, Change from posthog.models.dashboard import Dashboard from posthog.models.exported_asset import ExportedAsset, asset_for_token, get_content_response from posthog.models.insight import Insight -from posthog.models.session_recording import SessionRecording +from posthog.models import SessionRecording from posthog.models.user import User from posthog.permissions import ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission +from posthog.session_recordings.session_recording_api import SessionRecordingSerializer from posthog.user_permissions import UserPermissions from posthog.utils import render_template diff --git a/posthog/celery.py b/posthog/celery.py index 89d630a966ee5..7ed47f2503639 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -898,7 +898,7 @@ def check_flags_to_rollback(): @app.task(ignore_result=True) def ee_persist_single_recording(id: str, team_id: int): try: - from ee.tasks.session_recording.persistence import persist_single_recording + from ee.session_recordings.persistence_tasks import persist_single_recording persist_single_recording(id, team_id) except ImportError: @@ -908,7 +908,7 @@ def ee_persist_single_recording(id: str, team_id: int): @app.task(ignore_result=True) def ee_persist_finished_recordings(): try: - from ee.tasks.session_recording.persistence import persist_finished_recordings + from ee.session_recordings.persistence_tasks import persist_finished_recordings except ImportError: pass else: diff --git a/posthog/clickhouse/migrations/0006_session_recording_events.py b/posthog/clickhouse/migrations/0006_session_recording_events.py index 33a1765165741..5f9f1a8212261 100644 --- a/posthog/clickhouse/migrations/0006_session_recording_events.py +++ b/posthog/clickhouse/migrations/0006_session_recording_events.py @@ -1,5 +1,5 @@ from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions -from posthog.models.session_recording_event.sql import ( +from posthog.session_recordings.sql.session_recording_event_sql import ( DISTRIBUTED_SESSION_RECORDING_EVENTS_TABLE_SQL, KAFKA_SESSION_RECORDING_EVENTS_TABLE_SQL, SESSION_RECORDING_EVENTS_TABLE_MV_SQL, diff --git a/posthog/clickhouse/migrations/0020_session_recording_events_window_id.py b/posthog/clickhouse/migrations/0020_session_recording_events_window_id.py index d8cffe6140833..6c5cf0c46649f 100644 --- a/posthog/clickhouse/migrations/0020_session_recording_events_window_id.py +++ b/posthog/clickhouse/migrations/0020_session_recording_events_window_id.py @@ -1,5 +1,5 @@ from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions -from posthog.models.session_recording_event.sql import ( +from posthog.session_recordings.sql.session_recording_event_sql import ( KAFKA_SESSION_RECORDING_EVENTS_TABLE_SQL, SESSION_RECORDING_EVENTS_TABLE_MV_SQL, ) diff --git a/posthog/clickhouse/migrations/0036_session_recording_events_materialized_columns.py b/posthog/clickhouse/migrations/0036_session_recording_events_materialized_columns.py index 43a611ce4e2b1..be819a0111a01 100644 --- a/posthog/clickhouse/migrations/0036_session_recording_events_materialized_columns.py +++ b/posthog/clickhouse/migrations/0036_session_recording_events_materialized_columns.py @@ -1,7 +1,7 @@ from infi.clickhouse_orm import migrations from posthog.client import sync_execute -from posthog.models.session_recording_event.sql import MATERIALIZED_COLUMNS +from posthog.session_recordings.sql.session_recording_event_sql import MATERIALIZED_COLUMNS from posthog.settings import CLICKHOUSE_CLUSTER diff --git a/posthog/clickhouse/migrations/0043_session_replay_events.py b/posthog/clickhouse/migrations/0043_session_replay_events.py index 1fde598e3ea6c..32658df0e50e0 100644 --- a/posthog/clickhouse/migrations/0043_session_replay_events.py +++ b/posthog/clickhouse/migrations/0043_session_replay_events.py @@ -1,5 +1,5 @@ from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions -from posthog.models.session_replay_event.sql import ( +from posthog.session_recordings.sql.session_replay_event_sql import ( SESSION_REPLAY_EVENTS_TABLE_MV_SQL, KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, SESSION_REPLAY_EVENTS_TABLE_SQL, diff --git a/posthog/clickhouse/migrations/0044_session_replay_events_console_counts.py b/posthog/clickhouse/migrations/0044_session_replay_events_console_counts.py index b3d3b02fc8510..385165096a353 100644 --- a/posthog/clickhouse/migrations/0044_session_replay_events_console_counts.py +++ b/posthog/clickhouse/migrations/0044_session_replay_events_console_counts.py @@ -1,12 +1,12 @@ from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions -from posthog.models.session_replay_event.migrations_sql import ( +from posthog.session_recordings.sql.session_replay_event_migrations_sql import ( DROP_SESSION_REPLAY_EVENTS_TABLE_MV_SQL, DROP_KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, ADD_CONSOLE_COUNTS_WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL, ADD_CONSOLE_COUNTS_DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL, ADD_CONSOLE_COUNTS_SESSION_REPLAY_EVENTS_TABLE_SQL, ) -from posthog.models.session_replay_event.sql import ( +from posthog.session_recordings.sql.session_replay_event_sql import ( SESSION_REPLAY_EVENTS_TABLE_MV_SQL, KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, ) diff --git a/posthog/clickhouse/migrations/0045_session_replay_events_size.py b/posthog/clickhouse/migrations/0045_session_replay_events_size.py index 7f42278252b99..f09862de0bfc1 100644 --- a/posthog/clickhouse/migrations/0045_session_replay_events_size.py +++ b/posthog/clickhouse/migrations/0045_session_replay_events_size.py @@ -1,12 +1,12 @@ from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions -from posthog.models.session_replay_event.migrations_sql import ( +from posthog.session_recordings.sql.session_replay_event_migrations_sql import ( DROP_SESSION_REPLAY_EVENTS_TABLE_MV_SQL, DROP_KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, ADD_SIZE_WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL, ADD_SIZE_DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL, ADD_SIZE_SESSION_REPLAY_EVENTS_TABLE_SQL, ) -from posthog.models.session_replay_event.sql import ( +from posthog.session_recordings.sql.session_replay_event_sql import ( SESSION_REPLAY_EVENTS_TABLE_MV_SQL, KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, ) diff --git a/posthog/clickhouse/migrations/0048_session_replay_events_count.py b/posthog/clickhouse/migrations/0048_session_replay_events_count.py index d4676e2794884..2756f49ce2d46 100644 --- a/posthog/clickhouse/migrations/0048_session_replay_events_count.py +++ b/posthog/clickhouse/migrations/0048_session_replay_events_count.py @@ -1,12 +1,12 @@ from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions -from posthog.models.session_replay_event.migrations_sql import ( +from posthog.session_recordings.sql.session_replay_event_migrations_sql import ( DROP_SESSION_REPLAY_EVENTS_TABLE_MV_SQL, DROP_KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, ADD_EVENT_COUNT_WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL, ADD_EVENT_COUNT_DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL, ADD_EVENT_COUNT_SESSION_REPLAY_EVENTS_TABLE_SQL, ) -from posthog.models.session_replay_event.sql import ( +from posthog.session_recordings.sql.session_replay_event_sql import ( SESSION_REPLAY_EVENTS_TABLE_MV_SQL, KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, ) diff --git a/posthog/clickhouse/schema.py b/posthog/clickhouse/schema.py index aa31fb4343a63..b9e291092d0f1 100644 --- a/posthog/clickhouse/schema.py +++ b/posthog/clickhouse/schema.py @@ -27,8 +27,8 @@ PERSON_OVERRIDES_CREATE_MATERIALIZED_VIEW_SQL, PERSON_OVERRIDES_CREATE_TABLE_SQL, ) -from posthog.models.session_recording_event.sql import * -from posthog.models.session_replay_event.sql import ( +from posthog.session_recordings.sql.session_recording_event_sql import * +from posthog.session_recordings.sql.session_replay_event_sql import ( KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL, SESSION_REPLAY_EVENTS_TABLE_SQL, diff --git a/posthog/clickhouse/system_status.py b/posthog/clickhouse/system_status.py index 417525330a96c..b8cb1a6575970 100644 --- a/posthog/clickhouse/system_status.py +++ b/posthog/clickhouse/system_status.py @@ -11,7 +11,7 @@ from posthog.client import query_with_columns, sync_execute from posthog.cloud_utils import is_cloud from posthog.models.event.util import get_event_count, get_event_count_for_last_month, get_event_count_month_to_date -from posthog.models.session_recording_event.util import ( +from posthog.session_recordings.models.system_status_queries import ( get_recording_count_month_to_date, get_recording_events_count_month_to_date, ) diff --git a/posthog/conftest.py b/posthog/conftest.py index 0b9d3fd85c99c..06e7e256aed79 100644 --- a/posthog/conftest.py +++ b/posthog/conftest.py @@ -40,7 +40,7 @@ def reset_clickhouse_tables(): TRUNCATE_PERSON_STATIC_COHORT_TABLE_SQL, TRUNCATE_PERSON_TABLE_SQL, ) - from posthog.models.session_recording_event.sql import TRUNCATE_SESSION_RECORDING_EVENTS_TABLE_SQL + from posthog.session_recordings.sql.session_recording_event_sql import TRUNCATE_SESSION_RECORDING_EVENTS_TABLE_SQL # REMEMBER TO ADD ANY NEW CLICKHOUSE TABLES TO THIS ARRAY! TABLES_TO_CREATE_DROP = [ diff --git a/posthog/helpers/tests/test_session_recording_helpers.py b/posthog/helpers/tests/test_session_recording_helpers.py index 5561d0131f287..ee6a3c6ccada2 100644 --- a/posthog/helpers/tests/test_session_recording_helpers.py +++ b/posthog/helpers/tests/test_session_recording_helpers.py @@ -42,7 +42,7 @@ def mock_capture_flow(events: List[dict], max_size_bytes=512 * 1024) -> Tuple[Li ) new_replay_events = preprocess_replay_events_for_blob_ingestion(replay_events, max_size_bytes=max_size_bytes) - return (legacy_replay_events + other_events, new_replay_events + other_events) + return legacy_replay_events + other_events, new_replay_events + other_events def test_preprocess_with_no_recordings(): @@ -320,7 +320,7 @@ def test_decompress_data_returning_only_activity_info(chunked_and_compressed_sna def test_get_events_summary_from_snapshot_data(): timestamp = round(datetime.now().timestamp() * 1000) - snapshot_events = [ + snapshot_events: List[SnapshotData | None] = [ # ignore malformed events {"type": 2, "foo": "bar"}, # ignore other props diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index ed84eeaf4af6d..cb7d19f9df337 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -13,7 +13,7 @@ from posthog.models import Cohort from posthog.models.cohort.util import recalculate_cohortpeople from posthog.models.utils import UUIDT -from posthog.queries.session_recordings.test.session_replay_sql import produce_replay_summary +from posthog.session_recordings.queries.test.session_replay_sql import produce_replay_summary from posthog.schema import HogQLFilters, EventPropertyFilter, DateRange, QueryTiming from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_events from posthog.warehouse.models import DataWarehouseSavedQuery, DataWarehouseViewLink diff --git a/posthog/models/__init__.py b/posthog/models/__init__.py index 82679097f6ca2..89432e0809984 100644 --- a/posthog/models/__init__.py +++ b/posthog/models/__init__.py @@ -10,6 +10,7 @@ BatchExportDestination, BatchExportRun, ) +from ..warehouse.models import DataWarehouseTable from .cohort import Cohort, CohortPeople from .dashboard import Dashboard from .dashboard_tile import DashboardTile, Text @@ -36,13 +37,10 @@ from .organization_domain import OrganizationDomain from .person import Person, PersonDistinctId, PersonOverride, PersonOverrideMapping from .personal_api_key import PersonalAPIKey -from .plugin import Plugin, PluginAttachment, PluginConfig, PluginSourceFile +from .plugin import Plugin, PluginAttachment, PluginConfig, PluginSourceFile, PluginLogEntry from .prompt.prompt import Prompt, PromptSequence, UserPromptState from .property import Property from .property_definition import PropertyDefinition -from .session_recording import SessionRecording -from .session_recording_playlist import SessionRecordingPlaylist -from .session_recording_playlist_item import SessionRecordingPlaylistItem from .sharing_configuration import SharingConfiguration from .subscription import Subscription from .feedback.survey import Survey @@ -52,6 +50,9 @@ from .uploaded_media import UploadedMedia from .user import User, UserManager from .user_scene_personalisation import UserScenePersonalisation +from ..session_recordings.models.session_recording import SessionRecording +from ..session_recordings.models.session_recording_playlist import SessionRecordingPlaylist +from ..session_recordings.models.session_recording_playlist_item import SessionRecordingPlaylistItem __all__ = [ "Action", diff --git a/posthog/models/property/util.py b/posthog/models/property/util.py index 9dd81be1b3a5b..18368ac082f5d 100644 --- a/posthog/models/property/util.py +++ b/posthog/models/property/util.py @@ -48,7 +48,7 @@ from posthog.models.property.property import ValueT from posthog.models.team.team import groups_on_events_querying_enabled from posthog.queries.person_distinct_id_query import get_team_distinct_ids_query -from posthog.queries.session_query import SessionQuery +from posthog.session_recordings.queries.session_query import SessionQuery from posthog.queries.util import PersonPropertiesMode from posthog.utils import is_json, is_valid_regex diff --git a/posthog/models/session_recording/__init__.py b/posthog/models/session_recording/__init__.py deleted file mode 100644 index de4d98f29d25e..0000000000000 --- a/posthog/models/session_recording/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .session_recording import * diff --git a/posthog/models/session_recording_event/__init__.py b/posthog/models/session_recording_event/__init__.py deleted file mode 100644 index 23eedb489e29a..0000000000000 --- a/posthog/models/session_recording_event/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .session_recording_event import * diff --git a/posthog/models/session_recording_playlist/__init__.py b/posthog/models/session_recording_playlist/__init__.py deleted file mode 100644 index 29fa449f2cff0..0000000000000 --- a/posthog/models/session_recording_playlist/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .session_recording_playlist import * diff --git a/posthog/models/session_recording_playlist_item/__init__.py b/posthog/models/session_recording_playlist_item/__init__.py deleted file mode 100644 index 677b89fc1399d..0000000000000 --- a/posthog/models/session_recording_playlist_item/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .session_recording_playlist_item import * diff --git a/posthog/queries/breakdown_props.py b/posthog/queries/breakdown_props.py index 4e179cf4628db..9d0ccf80db32e 100644 --- a/posthog/queries/breakdown_props.py +++ b/posthog/queries/breakdown_props.py @@ -25,7 +25,7 @@ from posthog.queries.person_on_events_v2_sql import PERSON_OVERRIDES_JOIN_SQL from posthog.queries.person_query import PersonQuery from posthog.queries.query_date_range import QueryDateRange -from posthog.queries.session_query import SessionQuery +from posthog.session_recordings.queries.session_query import SessionQuery from posthog.queries.trends.sql import HISTOGRAM_ELEMENTS_ARRAY_OF_KEY_SQL, TOP_ELEMENTS_ARRAY_OF_KEY_SQL from posthog.queries.util import PersonPropertiesMode from posthog.utils import PersonOnEventsMode diff --git a/posthog/queries/event_query/event_query.py b/posthog/queries/event_query/event_query.py index aca34a3484cd7..9be4dc1a2fbd2 100644 --- a/posthog/queries/event_query/event_query.py +++ b/posthog/queries/event_query/event_query.py @@ -18,7 +18,7 @@ from posthog.queries.person_distinct_id_query import get_team_distinct_ids_query from posthog.queries.person_query import PersonQuery from posthog.queries.query_date_range import QueryDateRange -from posthog.queries.session_query import SessionQuery +from posthog.session_recordings.queries.session_query import SessionQuery from posthog.queries.util import PersonPropertiesMode from posthog.utils import PersonOnEventsMode from posthog.queries.person_on_events_v2_sql import PERSON_OVERRIDES_JOIN_SQL diff --git a/posthog/queries/trends/breakdown.py b/posthog/queries/trends/breakdown.py index 7fe281a0c158c..7e1d8c0b6198b 100644 --- a/posthog/queries/trends/breakdown.py +++ b/posthog/queries/trends/breakdown.py @@ -38,7 +38,7 @@ from posthog.queries.person_distinct_id_query import get_team_distinct_ids_query from posthog.queries.person_query import PersonQuery from posthog.queries.query_date_range import TIME_IN_SECONDS, QueryDateRange -from posthog.queries.session_query import SessionQuery +from posthog.session_recordings.queries.session_query import SessionQuery from posthog.queries.trends.sql import ( BREAKDOWN_ACTIVE_USER_AGGREGATE_SQL, BREAKDOWN_ACTIVE_USER_CONDITIONS_SQL, diff --git a/posthog/session_recordings/__init__.py b/posthog/session_recordings/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/posthog/session_recordings/models/__init__.py b/posthog/session_recordings/models/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/posthog/models/session_recording/metadata.py b/posthog/session_recordings/models/metadata.py similarity index 100% rename from posthog/models/session_recording/metadata.py rename to posthog/session_recordings/models/metadata.py diff --git a/posthog/models/session_recording/session_recording.py b/posthog/session_recordings/models/session_recording.py similarity index 96% rename from posthog/models/session_recording/session_recording.py rename to posthog/session_recordings/models/session_recording.py index f8eded26d9c5b..65dd13b913257 100644 --- a/posthog/models/session_recording/session_recording.py +++ b/posthog/session_recordings/models/session_recording.py @@ -6,15 +6,15 @@ from posthog.celery import ee_persist_single_recording from posthog.models.person.person import Person -from posthog.models.session_recording.metadata import ( +from posthog.session_recordings.models.metadata import ( DecompressedRecordingData, RecordingMatchingEvents, RecordingMetadata, ) -from posthog.models.session_recording_event.session_recording_event import SessionRecordingViewed +from posthog.session_recordings.models.session_recording_event import SessionRecordingViewed from posthog.models.team.team import Team from posthog.models.utils import UUIDModel -from posthog.queries.session_recordings.session_replay_events import SessionReplayEvents +from posthog.session_recordings.queries.session_replay_events import SessionReplayEvents from django.conf import settings @@ -98,7 +98,7 @@ def load_metadata(self) -> bool: return True def load_snapshots(self, limit=20, offset=0) -> None: - from posthog.queries.session_recordings.session_recording_events import SessionRecordingEvents + from posthog.session_recordings.queries.session_recording_events import SessionRecordingEvents if self._snapshots: return @@ -117,7 +117,7 @@ def load_object_data(self) -> None: This is only called in the to-be deprecated v1 of session recordings snapshot API """ try: - from ee.models.session_recording_extensions import load_persisted_recording + from ee.session_recordings.session_recording_extensions import load_persisted_recording except ImportError: load_persisted_recording = lambda *args: None diff --git a/posthog/models/session_recording_event/session_recording_event.py b/posthog/session_recordings/models/session_recording_event.py similarity index 100% rename from posthog/models/session_recording_event/session_recording_event.py rename to posthog/session_recordings/models/session_recording_event.py diff --git a/posthog/models/session_recording_playlist/session_recording_playlist.py b/posthog/session_recordings/models/session_recording_playlist.py similarity index 100% rename from posthog/models/session_recording_playlist/session_recording_playlist.py rename to posthog/session_recordings/models/session_recording_playlist.py diff --git a/posthog/models/session_recording_playlist_item/session_recording_playlist_item.py b/posthog/session_recordings/models/session_recording_playlist_item.py similarity index 100% rename from posthog/models/session_recording_playlist_item/session_recording_playlist_item.py rename to posthog/session_recordings/models/session_recording_playlist_item.py diff --git a/posthog/models/session_recording_event/util.py b/posthog/session_recordings/models/system_status_queries.py similarity index 100% rename from posthog/models/session_recording_event/util.py rename to posthog/session_recordings/models/system_status_queries.py diff --git a/posthog/session_recordings/queries/__init__.py b/posthog/session_recordings/queries/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/posthog/queries/session_query.py b/posthog/session_recordings/queries/session_query.py similarity index 100% rename from posthog/queries/session_query.py rename to posthog/session_recordings/queries/session_query.py diff --git a/posthog/queries/session_recordings/session_recording_events.py b/posthog/session_recordings/queries/session_recording_events.py similarity index 98% rename from posthog/queries/session_recordings/session_recording_events.py rename to posthog/session_recordings/queries/session_recording_events.py index 7a6975c3ef69d..826fb2a770ab0 100644 --- a/posthog/queries/session_recordings/session_recording_events.py +++ b/posthog/session_recordings/queries/session_recording_events.py @@ -4,7 +4,7 @@ from posthog.client import sync_execute from posthog.models import Team -from posthog.models.session_recording.metadata import ( +from posthog.session_recordings.models.metadata import ( DecompressedRecordingData, SessionRecordingEvent, SnapshotDataTaggedWithWindowId, diff --git a/posthog/queries/session_recordings/session_recording_list_from_replay_summary.py b/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py similarity index 100% rename from posthog/queries/session_recordings/session_recording_list_from_replay_summary.py rename to posthog/session_recordings/queries/session_recording_list_from_replay_summary.py diff --git a/posthog/queries/session_recordings/session_recording_properties.py b/posthog/session_recordings/queries/session_recording_properties.py similarity index 100% rename from posthog/queries/session_recordings/session_recording_properties.py rename to posthog/session_recordings/queries/session_recording_properties.py diff --git a/posthog/queries/session_recordings/session_replay_events.py b/posthog/session_recordings/queries/session_replay_events.py similarity index 97% rename from posthog/queries/session_recordings/session_replay_events.py rename to posthog/session_recordings/queries/session_replay_events.py index 3523be4e47101..6521e9f39fdb2 100644 --- a/posthog/queries/session_recordings/session_replay_events.py +++ b/posthog/session_recordings/queries/session_replay_events.py @@ -3,7 +3,7 @@ from posthog.clickhouse.client import sync_execute from posthog.models.team import Team -from posthog.models.session_recording.metadata import ( +from posthog.session_recordings.models.metadata import ( RecordingMetadata, ) diff --git a/posthog/session_recordings/queries/test/__init__.py b/posthog/session_recordings/queries/test/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list_from_session_replay.ambr b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr similarity index 100% rename from posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list_from_session_replay.ambr rename to posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr diff --git a/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_properties.ambr b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_properties.ambr similarity index 100% rename from posthog/queries/session_recordings/test/__snapshots__/test_session_recording_properties.ambr rename to posthog/session_recordings/queries/test/__snapshots__/test_session_recording_properties.ambr diff --git a/posthog/queries/session_recordings/test/__snapshots__/test_session_replay_summaries.ambr b/posthog/session_recordings/queries/test/__snapshots__/test_session_replay_summaries.ambr similarity index 100% rename from posthog/queries/session_recordings/test/__snapshots__/test_session_replay_summaries.ambr rename to posthog/session_recordings/queries/test/__snapshots__/test_session_replay_summaries.ambr diff --git a/posthog/queries/session_recordings/test/session_replay_sql.py b/posthog/session_recordings/queries/test/session_replay_sql.py similarity index 100% rename from posthog/queries/session_recordings/test/session_replay_sql.py rename to posthog/session_recordings/queries/test/session_replay_sql.py diff --git a/posthog/queries/session_recordings/test/test_session_recording.py b/posthog/session_recordings/queries/test/test_session_recording.py similarity index 90% rename from posthog/queries/session_recordings/test/test_session_recording.py rename to posthog/session_recordings/queries/test/test_session_recording.py index 7f55a50eb3cf2..28f992fabebd8 100644 --- a/posthog/queries/session_recordings/test/test_session_recording.py +++ b/posthog/session_recordings/queries/test/test_session_recording.py @@ -8,7 +8,7 @@ from posthog.models import Filter from posthog.models.team import Team -from posthog.queries.session_recordings.session_recording_events import SessionRecordingEvents +from posthog.session_recordings.queries.session_recording_events import SessionRecordingEvents from posthog.session_recordings.session_recording_helpers import ( DecompressedRecordingData, ) @@ -75,10 +75,11 @@ def test_get_snapshots(self): ) filter = create_recording_filter("1") - recording: DecompressedRecordingData = SessionRecordingEvents( + recording: DecompressedRecordingData | None = SessionRecordingEvents( team=self.team, session_recording_id="1" ).get_snapshots(filter.limit, filter.offset) + assert recording is not None self.assertEqual( recording["snapshot_data_by_window_id"], { @@ -116,10 +117,11 @@ def test_get_snapshots_does_not_leak_teams(self): ) filter = create_recording_filter("1") - recording: DecompressedRecordingData = SessionRecordingEvents( + recording: DecompressedRecordingData | None = SessionRecordingEvents( team=self.team, session_recording_id="1" ).get_snapshots(filter.limit, filter.offset) + assert recording is not None self.assertEqual( recording["snapshot_data_by_window_id"], {"": [{"data": {"source": 0}, "timestamp": 1600000000000, "type": 3}]}, @@ -127,10 +129,11 @@ def test_get_snapshots_does_not_leak_teams(self): def test_get_snapshots_with_no_such_session(self): filter = create_recording_filter("xxx") - recording: DecompressedRecordingData = SessionRecordingEvents( + recording: DecompressedRecordingData | None = SessionRecordingEvents( team=self.team, session_recording_id="xxx" ).get_snapshots(filter.limit, filter.offset) - assert not recording + + assert recording is None def test_get_chunked_snapshots(self): with freeze_time("2020-09-13T12:26:40.000Z"): @@ -149,9 +152,11 @@ def test_get_chunked_snapshots(self): ) filter = create_recording_filter(chunked_session_id) - recording: DecompressedRecordingData = SessionRecordingEvents( + recording: DecompressedRecordingData | None = SessionRecordingEvents( team=self.team, session_recording_id=chunked_session_id ).get_snapshots(limit, filter.offset) + + assert recording is not None self.assertEqual(len(recording["snapshot_data_by_window_id"][""]), limit * snapshots_per_chunk) self.assertTrue(recording["has_next"]) @@ -173,10 +178,11 @@ def test_get_chunked_snapshots_with_specific_limit_and_offset(self): ) filter = create_recording_filter(chunked_session_id, limit, offset) - recording: DecompressedRecordingData = SessionRecordingEvents( + recording: DecompressedRecordingData | None = SessionRecordingEvents( team=self.team, session_recording_id=chunked_session_id ).get_snapshots(limit, filter.offset) + assert recording is not None self.assertEqual(len(recording["snapshot_data_by_window_id"][""]), limit * snapshots_per_chunk) self.assertEqual(recording["snapshot_data_by_window_id"][""][0]["timestamp"], 1_600_000_300_000) self.assertTrue(recording["has_next"]) @@ -207,8 +213,9 @@ def test_get_snapshots_with_date_filter(self): filter = create_recording_filter( "1", ) - recording: DecompressedRecordingData = SessionRecordingEvents( + recording: DecompressedRecordingData | None = SessionRecordingEvents( team=self.team, session_recording_id="1", recording_start_time=now() ).get_snapshots(filter.limit, filter.offset) + assert recording is not None self.assertEqual(len(recording["snapshot_data_by_window_id"][""]), 1) diff --git a/posthog/session_recordings/queries/test/test_session_recording_list.py b/posthog/session_recordings/queries/test/test_session_recording_list.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/posthog/queries/session_recordings/test/test_session_recording_list_from_session_replay.py b/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py similarity index 99% rename from posthog/queries/session_recordings/test/test_session_recording_list_from_session_replay.py rename to posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py index 88484f316e150..6a529313a3851 100644 --- a/posthog/queries/session_recordings/test/test_session_recording_list_from_session_replay.py +++ b/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py @@ -12,13 +12,13 @@ from posthog.models.action import Action from posthog.models.action_step import ActionStep from posthog.models.filters.session_recordings_filter import SessionRecordingsFilter -from posthog.models.session_replay_event.sql import TRUNCATE_SESSION_REPLAY_EVENTS_TABLE_SQL +from posthog.session_recordings.sql.session_replay_event_sql import TRUNCATE_SESSION_REPLAY_EVENTS_TABLE_SQL from posthog.models.team import Team -from posthog.queries.session_recordings.session_recording_list_from_replay_summary import ( +from posthog.session_recordings.queries.session_recording_list_from_replay_summary import ( SessionRecordingListFromReplaySummary, ttl_days, ) -from posthog.queries.session_recordings.test.session_replay_sql import produce_replay_summary +from posthog.session_recordings.queries.test.session_replay_sql import produce_replay_summary from posthog.test.base import ( APIBaseTest, ClickhouseTestMixin, diff --git a/posthog/queries/session_recordings/test/test_session_recording_properties.py b/posthog/session_recordings/queries/test/test_session_recording_properties.py similarity index 98% rename from posthog/queries/session_recordings/test/test_session_recording_properties.py rename to posthog/session_recordings/queries/test/test_session_recording_properties.py index 1d553541fe5fa..387d41bbe1ebc 100644 --- a/posthog/queries/session_recordings/test/test_session_recording_properties.py +++ b/posthog/session_recordings/queries/test/test_session_recording_properties.py @@ -4,7 +4,7 @@ from posthog.models import Person from posthog.models.filters.session_recordings_filter import SessionRecordingsFilter -from posthog.queries.session_recordings.session_recording_properties import SessionRecordingProperties +from posthog.session_recordings.queries.session_recording_properties import SessionRecordingProperties from posthog.session_recordings.test.test_factory import create_snapshot from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, snapshot_clickhouse_queries diff --git a/posthog/queries/session_recordings/test/test_session_replay_events.py b/posthog/session_recordings/queries/test/test_session_replay_events.py similarity index 95% rename from posthog/queries/session_recordings/test/test_session_replay_events.py rename to posthog/session_recordings/queries/test/test_session_replay_events.py index 873e741b3fc48..c304233ff98d4 100644 --- a/posthog/queries/session_recordings/test/test_session_replay_events.py +++ b/posthog/session_recordings/queries/test/test_session_replay_events.py @@ -1,6 +1,6 @@ from posthog.models import Team -from posthog.queries.session_recordings.session_replay_events import SessionReplayEvents -from posthog.queries.session_recordings.test.session_replay_sql import produce_replay_summary +from posthog.session_recordings.queries.session_replay_events import SessionReplayEvents +from posthog.session_recordings.queries.test.session_replay_sql import produce_replay_summary from posthog.test.base import ClickhouseTestMixin, APIBaseTest from dateutil.relativedelta import relativedelta from django.utils.timezone import now diff --git a/posthog/queries/session_recordings/test/test_session_replay_summaries.py b/posthog/session_recordings/queries/test/test_session_replay_summaries.py similarity index 98% rename from posthog/queries/session_recordings/test/test_session_replay_summaries.py rename to posthog/session_recordings/queries/test/test_session_replay_summaries.py index 0b3e361fa9511..5a1e9b94db842 100644 --- a/posthog/queries/session_recordings/test/test_session_replay_summaries.py +++ b/posthog/session_recordings/queries/test/test_session_replay_summaries.py @@ -9,7 +9,7 @@ from posthog.models import Team from posthog.models.event.util import format_clickhouse_timestamp from posthog.queries.app_metrics.serializers import AppMetricsRequestSerializer -from posthog.queries.session_recordings.test.session_replay_sql import produce_replay_summary +from posthog.session_recordings.queries.test.session_replay_sql import produce_replay_summary from posthog.test.base import BaseTest, ClickhouseTestMixin, snapshot_clickhouse_queries diff --git a/posthog/session_recordings/realtime_snapshots.py b/posthog/session_recordings/realtime_snapshots.py index ea19b3b405a2b..20e8a0846440c 100644 --- a/posthog/session_recordings/realtime_snapshots.py +++ b/posthog/session_recordings/realtime_snapshots.py @@ -72,8 +72,8 @@ def get_realtime_snapshots(team_id: str, session_id: str, attempt_count=0) -> Op except Exception as e: # very broad capture to see if there are any unexpected errors capture_exception( - "get_realtime_snapshots_failed", - extras={"attempt_count": attempt_count}, + e, + extras={"attempt_count": attempt_count, "operation": "get_realtime_snapshots"}, tags={"team_id": team_id, "session_id": session_id}, ) raise e diff --git a/posthog/api/session_recording.py b/posthog/session_recordings/session_recording_api.py similarity index 98% rename from posthog/api/session_recording.py rename to posthog/session_recordings/session_recording_api.py index a5e5bf7fb937a..41a668083b534 100644 --- a/posthog/api/session_recording.py +++ b/posthog/session_recordings/session_recording_api.py @@ -25,19 +25,19 @@ from posthog.models import Filter, User from posthog.models.filters.session_recordings_filter import SessionRecordingsFilter from posthog.models.person.person import PersonDistinctId -from posthog.models.session_recording.session_recording import SessionRecording -from posthog.models.session_recording_event import SessionRecordingViewed +from posthog.session_recordings.models.session_recording import SessionRecording from posthog.permissions import ( ProjectMembershipNecessaryPermissions, SharingTokenPermission, TeamMemberAccessPermission, ) +from posthog.session_recordings.models.session_recording_event import SessionRecordingViewed -from posthog.queries.session_recordings.session_recording_list_from_replay_summary import ( +from posthog.session_recordings.queries.session_recording_list_from_replay_summary import ( SessionRecordingListFromReplaySummary, SessionIdEventsQuery, ) -from posthog.queries.session_recordings.session_recording_properties import SessionRecordingProperties +from posthog.session_recordings.queries.session_recording_properties import SessionRecordingProperties from posthog.rate_limit import ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle from posthog.session_recordings.realtime_snapshots import get_realtime_snapshots from posthog.storage import object_storage @@ -317,7 +317,7 @@ def _snapshots_v2(self, request: request.Request): response_data["sources"] = sources elif source == "realtime": - snapshots = get_realtime_snapshots(team_id=self.team.pk, session_id=recording.session_id) or [] + snapshots = get_realtime_snapshots(team_id=self.team.pk, session_id=str(recording.session_id)) or [] event_properties["source"] = "realtime" event_properties["snapshots_length"] = len(snapshots) diff --git a/posthog/session_recordings/session_recording_helpers.py b/posthog/session_recordings/session_recording_helpers.py index 03cb4e71dbd8f..960ac0021c817 100644 --- a/posthog/session_recordings/session_recording_helpers.py +++ b/posthog/session_recordings/session_recording_helpers.py @@ -3,13 +3,13 @@ import json from collections import defaultdict from datetime import datetime, timezone -from typing import Any, Callable, DefaultDict, Dict, Generator, List, Optional +from typing import Any, Callable, DefaultDict, Dict, Generator, List, Optional, Tuple from dateutil.parser import ParserError, parse from sentry_sdk.api import capture_exception from posthog.models import utils -from posthog.models.session_recording.metadata import ( +from posthog.session_recordings.models.metadata import ( DecompressedRecordingData, SessionRecordingEventSummary, SnapshotData, @@ -126,7 +126,7 @@ def legacy_compress_and_chunk_snapshots(events: List[Event], chunk_size=512 * 10 } -def split_replay_events(events: List[Event]) -> List[Event]: +def split_replay_events(events: List[Event]) -> Tuple[List[Event], List[Event]]: replay, other = [], [] for event in events: @@ -139,7 +139,9 @@ def preprocess_replay_events_for_blob_ingestion(events: List[Event], max_size_by return _process_windowed_events(events, lambda x: preprocess_replay_events(x, max_size_bytes=max_size_bytes)) -def preprocess_replay_events(events: List[Event], max_size_bytes=1024 * 1024) -> List[Event]: +def preprocess_replay_events( + _events: List[Event] | Generator[Event, None, None], max_size_bytes=1024 * 1024 +) -> Generator[Event, None, None]: """ The events going to blob ingestion are uncompressed (the compression happens in the Kafka producer) 1. Since posthog-js {version} we are grouping events on the frontend in a batch and passing their size in $snapshot_bytes @@ -149,8 +151,14 @@ def preprocess_replay_events(events: List[Event], max_size_bytes=1024 * 1024) -> 3. If not, we split out the "full snapshots" from the rest (they are typically bigger) and send them individually, trying one more time to group the rest, otherwise sending them individually """ + if isinstance(_events, Generator): + # we check the first item in the events below so need to be dealing with a list + events = list(_events) + else: + events = _events + if len(events) == 0: - return [] + return size_with_headroom = max_size_bytes * 0.95 # Leave 5% headroom @@ -158,7 +166,7 @@ def preprocess_replay_events(events: List[Event], max_size_bytes=1024 * 1024) -> session_id = events[0]["properties"]["$session_id"] window_id = events[0]["properties"].get("$window_id") - def new_event(items: List[dict] = None) -> Event: + def new_event(items: List[dict] | None = None) -> Event: return { **events[0], "event": "$snapshot_items", # New event name to avoid confusion with the old $snapshot event @@ -173,7 +181,7 @@ def new_event(items: List[dict] = None) -> Event: # 1. Group by $snapshot_bytes if any of the events have it if events[0]["properties"].get("$snapshot_bytes"): - current_event = None + current_event: Dict | None = None current_event_size = 0 for event in events: @@ -191,7 +199,8 @@ def new_event(items: List[dict] = None) -> Event: current_event["properties"]["$snapshot_items"].extend(additional_data) current_event_size += additional_bytes - yield current_event + if current_event: + yield current_event else: snapshot_data_list = list(flatten([event["properties"]["$snapshot_data"] for event in events], max_depth=1)) @@ -226,11 +235,13 @@ def new_event(items: List[dict] = None) -> Event: yield event -def _process_windowed_events(events: List[Event], fn: Callable[[List[Event], Any], List[Event]]) -> List[Event]: +def _process_windowed_events( + events: List[Event], fn: Callable[[List[Any]], Generator[Event, None, None]] +) -> List[Event]: """ Helper method to simplify grouping events by window_id and session_id, processing them with the given function, and then returning the flattened list """ - result = [] + result: List[Event] = [] snapshots_by_session_and_window_id = defaultdict(list) for event in events: @@ -315,7 +326,7 @@ def decompress_chunked_snapshot_data( event["snapshot_data"]["events_summary"] if return_only_activity_data else decompressed_items ) else: - # Really old format where the event is just a single raw rrweb event + # Old format where the event is just a single raw rrweb event snapshot_data_by_window_id[event["window_id"]].append( get_events_summary_from_snapshot_data([event["snapshot_data"]])[0] if return_only_activity_data @@ -339,7 +350,7 @@ def decompress_chunked_snapshot_data( if len(chunks) == event["snapshot_data"]["chunk_count"]: count += 1 - chunks_collector[event["snapshot_data"]["chunk_id"]] = None + chunks_collector[event["snapshot_data"]["chunk_id"]] = [] # Somehow mark this chunk_id as processed... processed_chunk_ids.add(event["snapshot_data"]["chunk_id"]) @@ -395,7 +406,9 @@ def convert_to_timestamp(source: str) -> int: return int(parse(source).timestamp() * 1000) -def get_events_summary_from_snapshot_data(snapshot_data: List[SnapshotData]) -> List[SessionRecordingEventSummary]: +def get_events_summary_from_snapshot_data( + snapshot_data: List[SnapshotData | None], +) -> List[SessionRecordingEventSummary]: """ Extract a minimal representation of the snapshot data events for easier querying. 'data' and 'data.payload' values are included as long as they are strings or numbers diff --git a/posthog/session_recordings/sql/__init__.py b/posthog/session_recordings/sql/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/posthog/models/session_recording_event/sql.py b/posthog/session_recordings/sql/session_recording_event_sql.py similarity index 100% rename from posthog/models/session_recording_event/sql.py rename to posthog/session_recordings/sql/session_recording_event_sql.py diff --git a/posthog/models/session_replay_event/migrations_sql.py b/posthog/session_recordings/sql/session_replay_event_migrations_sql.py similarity index 97% rename from posthog/models/session_replay_event/migrations_sql.py rename to posthog/session_recordings/sql/session_replay_event_migrations_sql.py index b11f5581c930f..ac897fccc1d08 100644 --- a/posthog/models/session_replay_event/migrations_sql.py +++ b/posthog/session_recordings/sql/session_replay_event_migrations_sql.py @@ -1,6 +1,6 @@ from django.conf import settings -from posthog.models.session_replay_event.sql import SESSION_REPLAY_EVENTS_DATA_TABLE +from posthog.session_recordings.sql.session_replay_event_sql import SESSION_REPLAY_EVENTS_DATA_TABLE DROP_SESSION_REPLAY_EVENTS_TABLE_MV_SQL = ( lambda: "DROP TABLE IF EXISTS session_replay_events_mv ON CLUSTER {cluster}".format( diff --git a/posthog/models/session_replay_event/sql.py b/posthog/session_recordings/sql/session_replay_event_sql.py similarity index 100% rename from posthog/models/session_replay_event/sql.py rename to posthog/session_recordings/sql/session_replay_event_sql.py diff --git a/posthog/session_recordings/test/__init__.py b/posthog/session_recordings/test/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/posthog/api/test/__snapshots__/test_session_recordings.ambr b/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr similarity index 94% rename from posthog/api/test/__snapshots__/test_session_recordings.ambr rename to posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr index e97965ce54b8f..a9f66096059e3 100644 --- a/posthog/api/test/__snapshots__/test_session_recordings.ambr +++ b/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr @@ -119,10 +119,10 @@ FROM "posthog_sessionrecording" LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") WHERE ("posthog_sessionrecording"."session_id" IN ('5', - '2', - '3', '4', - '1') + '1', + '3', + '2') AND "posthog_sessionrecording"."team_id" = 2) GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' @@ -477,11 +477,11 @@ FROM "posthog_sessionrecording" LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") WHERE ("posthog_sessionrecording"."session_id" IN ('5', - '2', - '3', '4', - '6', - '1') + '1', + '3', + '2', + '6') AND "posthog_sessionrecording"."team_id" = 2) GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' @@ -848,12 +848,12 @@ FROM "posthog_sessionrecording" LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") WHERE ("posthog_sessionrecording"."session_id" IN ('5', - '2', - '3', '4', + '1', + '3', + '2', '7', - '6', - '1') + '6') AND "posthog_sessionrecording"."team_id" = 2) GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' @@ -1258,13 +1258,13 @@ FROM "posthog_sessionrecording" LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") WHERE ("posthog_sessionrecording"."session_id" IN ('5', - '2', - '3', '4', - '7', - '6', '1', - '8') + '8', + '3', + '2', + '7', + '6') AND "posthog_sessionrecording"."team_id" = 2) GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' @@ -1684,14 +1684,14 @@ FROM "posthog_sessionrecording" LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") WHERE ("posthog_sessionrecording"."session_id" IN ('5', - '2', - '3', '4', - '7', - '6', '1', '8', - '9') + '3', + '2', + '7', + '9', + '6') AND "posthog_sessionrecording"."team_id" = 2) GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' @@ -2079,15 +2079,15 @@ FROM "posthog_sessionrecording" LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") WHERE ("posthog_sessionrecording"."session_id" IN ('5', - '2', - '3', '4', - '7', - '10', - '6', '1', '8', - '9') + '3', + '2', + '7', + '9', + '6', + '10') AND "posthog_sessionrecording"."team_id" = 2) GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' @@ -2147,61 +2147,6 @@ 5 /* ... */) /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' --- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.195 - ' - SELECT "posthog_instancesetting"."id", - "posthog_instancesetting"."key", - "posthog_instancesetting"."raw_value" - FROM "posthog_instancesetting" - WHERE "posthog_instancesetting"."key" = 'constance:posthog:PERSON_ON_EVENTS_V2_ENABLED' - ORDER BY "posthog_instancesetting"."id" ASC - LIMIT 1 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.196 - ' - SELECT "posthog_instancesetting"."id", - "posthog_instancesetting"."key", - "posthog_instancesetting"."raw_value" - FROM "posthog_instancesetting" - WHERE "posthog_instancesetting"."key" = 'constance:posthog:PERSON_ON_EVENTS_ENABLED' - ORDER BY "posthog_instancesetting"."id" ASC - LIMIT 1 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.197 - ' - SELECT "posthog_instancesetting"."id", - "posthog_instancesetting"."key", - "posthog_instancesetting"."raw_value" - FROM "posthog_instancesetting" - WHERE "posthog_instancesetting"."key" = 'constance:posthog:AGGREGATE_BY_DISTINCT_IDS_TEAMS' - ORDER BY "posthog_instancesetting"."id" ASC - LIMIT 1 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.198 - ' - SELECT "posthog_instancesetting"."id", - "posthog_instancesetting"."key", - "posthog_instancesetting"."raw_value" - FROM "posthog_instancesetting" - WHERE "posthog_instancesetting"."key" = 'constance:posthog:RECORDINGS_TTL_WEEKS' - ORDER BY "posthog_instancesetting"."id" ASC - LIMIT 1 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.199 - ' - SELECT "posthog_instancesetting"."id", - "posthog_instancesetting"."key", - "posthog_instancesetting"."raw_value" - FROM "posthog_instancesetting" - WHERE "posthog_instancesetting"."key" = 'constance:posthog:AGGREGATE_BY_DISTINCT_IDS_TEAMS' - ORDER BY "posthog_instancesetting"."id" ASC - LIMIT 1 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.2 ' SELECT "posthog_organizationmembership"."id", @@ -2242,122 +2187,6 @@ LIMIT 1 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' --- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.200 - ' - SELECT "posthog_instancesetting"."id", - "posthog_instancesetting"."key", - "posthog_instancesetting"."raw_value" - FROM "posthog_instancesetting" - WHERE "posthog_instancesetting"."key" = 'constance:posthog:RECORDINGS_TTL_WEEKS' - ORDER BY "posthog_instancesetting"."id" ASC - LIMIT 1 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.201 - ' - SELECT "posthog_instancesetting"."id", - "posthog_instancesetting"."key", - "posthog_instancesetting"."raw_value" - FROM "posthog_instancesetting" - WHERE "posthog_instancesetting"."key" = 'constance:posthog:AGGREGATE_BY_DISTINCT_IDS_TEAMS' - ORDER BY "posthog_instancesetting"."id" ASC - LIMIT 1 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.202 - ' - SELECT "posthog_sessionrecording"."id", - "posthog_sessionrecording"."session_id", - "posthog_sessionrecording"."team_id", - "posthog_sessionrecording"."created_at", - "posthog_sessionrecording"."deleted", - "posthog_sessionrecording"."object_storage_path", - "posthog_sessionrecording"."distinct_id", - "posthog_sessionrecording"."duration", - "posthog_sessionrecording"."active_seconds", - "posthog_sessionrecording"."inactive_seconds", - "posthog_sessionrecording"."start_time", - "posthog_sessionrecording"."end_time", - "posthog_sessionrecording"."click_count", - "posthog_sessionrecording"."keypress_count", - "posthog_sessionrecording"."mouse_activity_count", - "posthog_sessionrecording"."console_log_count", - "posthog_sessionrecording"."console_warn_count", - "posthog_sessionrecording"."console_error_count", - "posthog_sessionrecording"."start_url", - "posthog_sessionrecording"."storage_version", - COUNT("posthog_sessionrecordingplaylistitem"."id") AS "pinned_count" - FROM "posthog_sessionrecording" - LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") - WHERE ("posthog_sessionrecording"."session_id" IN ('5', - '2', - '3', - '4', - '7', - '10', - '6', - '1', - '8', - '9') - AND "posthog_sessionrecording"."team_id" = 2) - GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.203 - ' - SELECT "posthog_sessionrecordingviewed"."session_id" - FROM "posthog_sessionrecordingviewed" - WHERE ("posthog_sessionrecordingviewed"."team_id" = 2 - AND "posthog_sessionrecordingviewed"."user_id" = 2) /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.204 - ' - SELECT "posthog_persondistinctid"."id", - "posthog_persondistinctid"."team_id", - "posthog_persondistinctid"."person_id", - "posthog_persondistinctid"."distinct_id", - "posthog_persondistinctid"."version", - "posthog_person"."id", - "posthog_person"."created_at", - "posthog_person"."properties_last_updated_at", - "posthog_person"."properties_last_operation", - "posthog_person"."team_id", - "posthog_person"."properties", - "posthog_person"."is_user_id", - "posthog_person"."is_identified", - "posthog_person"."uuid", - "posthog_person"."version" - FROM "posthog_persondistinctid" - INNER JOIN "posthog_person" ON ("posthog_persondistinctid"."person_id" = "posthog_person"."id") - WHERE ("posthog_persondistinctid"."distinct_id" IN ('user1', - 'user10', - 'user2', - 'user3', - 'user4', - 'user5', - 'user6', - 'user7', - 'user8', - 'user9') - AND "posthog_persondistinctid"."team_id" = 2) /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- -# name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.205 - ' - SELECT "posthog_persondistinctid"."id", - "posthog_persondistinctid"."team_id", - "posthog_persondistinctid"."person_id", - "posthog_persondistinctid"."distinct_id", - "posthog_persondistinctid"."version" - FROM "posthog_persondistinctid" - WHERE "posthog_persondistinctid"."person_id" IN (1, - 2, - 3, - 4, - 5 /* ... */) /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ - ' ---- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.21 ' SELECT "posthog_instancesetting"."id", @@ -2832,8 +2661,8 @@ COUNT("posthog_sessionrecordingplaylistitem"."id") AS "pinned_count" FROM "posthog_sessionrecording" LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") - WHERE ("posthog_sessionrecording"."session_id" IN ('2', - '1') + WHERE ("posthog_sessionrecording"."session_id" IN ('1', + '2') AND "posthog_sessionrecording"."team_id" = 2) GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' @@ -3195,9 +3024,9 @@ COUNT("posthog_sessionrecordingplaylistitem"."id") AS "pinned_count" FROM "posthog_sessionrecording" LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") - WHERE ("posthog_sessionrecording"."session_id" IN ('2', + WHERE ("posthog_sessionrecording"."session_id" IN ('1', '3', - '1') + '2') AND "posthog_sessionrecording"."team_id" = 2) GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' @@ -3560,10 +3389,10 @@ COUNT("posthog_sessionrecordingplaylistitem"."id") AS "pinned_count" FROM "posthog_sessionrecording" LEFT OUTER JOIN "posthog_sessionrecordingplaylistitem" ON ("posthog_sessionrecording"."session_id" = "posthog_sessionrecordingplaylistitem"."recording_id") - WHERE ("posthog_sessionrecording"."session_id" IN ('2', + WHERE ("posthog_sessionrecording"."session_id" IN ('4', + '1', '3', - '4', - '1') + '2') AND "posthog_sessionrecording"."team_id" = 2) GROUP BY "posthog_sessionrecording"."id" /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ ' diff --git a/posthog/session_recordings/test/test_factory.py b/posthog/session_recordings/test/test_factory.py index f67b69febb531..4213ff02f5566 100644 --- a/posthog/session_recordings/test/test_factory.py +++ b/posthog/session_recordings/test/test_factory.py @@ -8,8 +8,8 @@ from posthog.client import sync_execute from posthog.kafka_client.client import ClickhouseProducer from posthog.kafka_client.topics import KAFKA_CLICKHOUSE_SESSION_RECORDING_EVENTS -from posthog.models.session_recording_event.sql import INSERT_SESSION_RECORDING_EVENT_SQL -from posthog.queries.session_recordings.test.session_replay_sql import produce_replay_summary +from posthog.session_recordings.sql.session_recording_event_sql import INSERT_SESSION_RECORDING_EVENT_SQL +from posthog.session_recordings.queries.test.session_replay_sql import produce_replay_summary from posthog.session_recordings.session_recording_helpers import ( RRWEB_MAP_EVENT_TYPE, legacy_preprocess_session_recording_events_for_clickhouse, diff --git a/posthog/api/test/test_session_recordings.py b/posthog/session_recordings/test/test_session_recordings.py similarity index 96% rename from posthog/api/test/test_session_recordings.py rename to posthog/session_recordings/test/test_session_recordings.py index 47bd7f0e716a1..e52a241cf3049 100644 --- a/posthog/api/test/test_session_recordings.py +++ b/posthog/session_recordings/test/test_session_recordings.py @@ -12,14 +12,13 @@ from freezegun import freeze_time from rest_framework import status -from posthog.api.session_recording import DEFAULT_RECORDING_CHUNK_LIMIT +from posthog.session_recordings.models.session_recording_event import SessionRecordingViewed from posthog.api.test.test_team import create_team from posthog.constants import SESSION_RECORDINGS_FILTER_IDS from posthog.models import Organization, Person, SessionRecording from posthog.models.filters.session_recordings_filter import SessionRecordingsFilter -from posthog.models.session_recording_event import SessionRecordingViewed from posthog.models.team import Team -from posthog.queries.session_recordings.test.session_replay_sql import produce_replay_summary +from posthog.session_recordings.queries.test.session_replay_sql import produce_replay_summary from posthog.session_recordings.test.test_factory import create_session_recording_events from posthog.test.base import ( APIBaseTest, @@ -170,7 +169,7 @@ def test_get_session_recordings(self): (session_id_one, "user", base_time, base_time + relativedelta(seconds=30), 30, False, user.pk), ] - @patch("posthog.api.session_recording.SessionRecordingListFromReplaySummary") + @patch("posthog.session_recordings.session_recording_api.SessionRecordingListFromReplaySummary") def test_console_log_filters_are_correctly_passed_to_listing(self, mock_summary_lister): mock_summary_lister.return_value.run.return_value = ([], False) @@ -371,6 +370,9 @@ def test_get_single_session_recording_metadata(self): } def test_get_default_limit_of_chunks(self): + # TODO import causes circular reference... but we're going to delete this soon so... + from posthog.session_recordings.session_recording_api import DEFAULT_RECORDING_CHUNK_LIMIT + base_time = now() num_snapshots = DEFAULT_RECORDING_CHUNK_LIMIT + 10 @@ -401,6 +403,9 @@ def test_get_snapshots_is_compressed(self): self.assertEqual(response.headers.get("Content-Encoding", None), "gzip") def test_get_snapshots_for_chunked_session_recording(self): + # TODO import causes circular reference... but we're going to delete this soon so... + from posthog.session_recordings.session_recording_api import DEFAULT_RECORDING_CHUNK_LIMIT + chunked_session_id = "chunk_id" expected_num_requests = 3 num_chunks = 60 @@ -561,7 +566,7 @@ def test_delete_session_recording(self): # New snapshot loading method @freeze_time("2023-01-01T00:00:00Z") - @patch("posthog.api.session_recording.object_storage.list_objects") + @patch("posthog.session_recordings.session_recording_api.object_storage.list_objects") def test_get_snapshots_v2_default_response(self, mock_list_objects) -> None: session_id = str(uuid.uuid4()) timestamp = round(now().timestamp() * 1000) @@ -597,7 +602,7 @@ def test_get_snapshots_v2_default_response(self, mock_list_objects) -> None: mock_list_objects.assert_called_with(f"session_recordings/team_id/{self.team.pk}/session_id/{session_id}/data") @freeze_time("2023-01-01T00:00:00Z") - @patch("posthog.api.session_recording.object_storage.list_objects") + @patch("posthog.session_recordings.session_recording_api.object_storage.list_objects") def test_get_snapshots_upgrade_to_v2_if_stored_recording_requires_it(self, mock_list_objects: MagicMock) -> None: session_id = str(uuid.uuid4()) timestamp = round(now().timestamp() * 1000) @@ -622,7 +627,7 @@ def test_get_snapshots_upgrade_to_v2_if_stored_recording_requires_it(self, mock_ mock_list_objects.assert_not_called() @freeze_time("2023-01-01T00:00:00Z") - @patch("posthog.api.session_recording.object_storage.list_objects") + @patch("posthog.session_recordings.session_recording_api.object_storage.list_objects") def test_get_snapshots_v2_from_lts(self, mock_list_objects: MagicMock) -> None: session_id = str(uuid.uuid4()) timestamp = round(now().timestamp() * 1000) @@ -679,7 +684,7 @@ def list_objects_func(path: str) -> List[str]: ] @freeze_time("2023-01-01T00:00:00Z") - @patch("posthog.api.session_recording.object_storage.list_objects") + @patch("posthog.session_recordings.session_recording_api.object_storage.list_objects") def test_get_snapshots_v2_default_response_no_realtime_if_old(self, mock_list_objects) -> None: session_id = str(uuid.uuid4()) old_timestamp = round((now() - timedelta(hours=26)).timestamp() * 1000) @@ -701,9 +706,9 @@ def test_get_snapshots_v2_default_response_no_realtime_if_old(self, mock_list_ob ] } - @patch("posthog.api.session_recording.SessionRecording.get_or_build") - @patch("posthog.api.session_recording.object_storage.get_presigned_url") - @patch("posthog.api.session_recording.requests") + @patch("posthog.session_recordings.session_recording_api.SessionRecording.get_or_build") + @patch("posthog.session_recordings.session_recording_api.object_storage.get_presigned_url") + @patch("posthog.session_recordings.session_recording_api.requests") def test_can_get_session_recording_blob( self, _mock_requests, mock_presigned_url, mock_get_session_recording ) -> None: @@ -726,9 +731,9 @@ def presigned_url_sideeffect(key: str, **kwargs): response = self.client.get(url) assert response.status_code == status.HTTP_200_OK - @patch("posthog.api.session_recording.SessionRecording.get_or_build") - @patch("posthog.api.session_recording.object_storage.get_presigned_url") - @patch("posthog.api.session_recording.requests") + @patch("posthog.session_recordings.session_recording_api.SessionRecording.get_or_build") + @patch("posthog.session_recordings.session_recording_api.object_storage.get_presigned_url") + @patch("posthog.session_recordings.session_recording_api.requests") def test_cannot_get_session_recording_blob_for_made_up_sessions( self, _mock_requests, mock_presigned_url, mock_get_session_recording ) -> None: @@ -744,7 +749,7 @@ def test_cannot_get_session_recording_blob_for_made_up_sessions( assert response.status_code == status.HTTP_404_NOT_FOUND assert mock_presigned_url.call_count == 0 - @patch("posthog.api.session_recording.object_storage.get_presigned_url") + @patch("posthog.session_recordings.session_recording_api.object_storage.get_presigned_url") def test_can_not_get_session_recording_blob_that_does_not_exist(self, mock_presigned_url) -> None: session_id = str(uuid.uuid4()) blob_key = f"session_recordings/team_id/{self.team.pk}/session_id/{session_id}/data/1682608337071" diff --git a/posthog/tasks/test/test_usage_report.py b/posthog/tasks/test/test_usage_report.py index 636b3e76b93e9..86ba59e07a8d8 100644 --- a/posthog/tasks/test/test_usage_report.py +++ b/posthog/tasks/test/test_usage_report.py @@ -221,7 +221,7 @@ def _create_sample_usage_data(self) -> None: create_snapshot( has_full_snapshot=True, distinct_id=distinct_id, - session_id=i, + session_id=str(i), timestamp=now() - relativedelta(hours=12), team_id=self.org_1_team_2.id, ) @@ -232,7 +232,7 @@ def _create_sample_usage_data(self) -> None: create_snapshot( has_full_snapshot=True, distinct_id=distinct_id, - session_id=i + 10, + session_id=str(i + 10), timestamp=now() - relativedelta(hours=48), team_id=self.org_1_team_2.id, ) diff --git a/posthog/test/base.py b/posthog/test/base.py index b4b60d36363c4..8b66387037a7c 100644 --- a/posthog/test/base.py +++ b/posthog/test/base.py @@ -39,12 +39,12 @@ TRUNCATE_PERSON_STATIC_COHORT_TABLE_SQL, ) from posthog.models.person.util import bulk_create_persons, create_person -from posthog.models.session_recording_event.sql import ( +from posthog.session_recordings.sql.session_recording_event_sql import ( DISTRIBUTED_SESSION_RECORDING_EVENTS_TABLE_SQL, DROP_SESSION_RECORDING_EVENTS_TABLE_SQL, SESSION_RECORDING_EVENTS_TABLE_SQL, ) -from posthog.models.session_replay_event.sql import ( +from posthog.session_recordings.sql.session_replay_event_sql import ( DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL, DROP_SESSION_REPLAY_EVENTS_TABLE_SQL, SESSION_REPLAY_EVENTS_TABLE_SQL,