Skip to content

Commit

Permalink
Merge branch 'master' into split-temporal-tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@Twixes
Twixes committed Oct 30, 2024
2 parents 299da94 + 78ea7fb commit 849b918
Show file tree
Hide file tree
Showing 13 changed files with 276 additions and 307 deletions.
4 changes: 4 additions & 0 deletions frontend/src/scenes/pipeline/hogfunctions/filters/HogFunctionFilters.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,10 @@ export function HogFunctionFilters(): JSX.Element {
value: '{person.id}',
label: 'Run once per person per interval',
},
{
value: '{concat(person.id, event.event)}',
label: 'Run once per person per event per interval',
},
]}
value={value?.hash ?? null}
onChange={(val) =>
Expand Down
12 changes: 10 additions & 2 deletions plugin-server/tests/cdp/examples.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -708,8 +708,16 @@ export const HOG_MASK_EXAMPLES: Record<string, Pick<HogFunctionType, 'masking'>>
person: {
masking: {
ttl: 30,
hash: '{person.uuid}',
bytecode: ['_h', 32, 'uuid', 32, 'person', 1, 2],
hash: '{person.id}',
bytecode: ['_h', 32, 'id', 32, 'person', 1, 2],
threshold: null,
},
},
personAndEvent: {
masking: {
ttl: 30,
hash: '{concat(person.id, event.event)}',
bytecode: ['_H', 1, 32, 'id', 32, 'person', 1, 2, 32, 'event', 32, 'event', 1, 2, 2, 'concat', 2],
threshold: null,
},
},
Expand Down
37 changes: 29 additions & 8 deletions plugin-server/tests/cdp/hog-masker.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ describe('HogMasker', () => {
describe('ttl', () => {
let hogFunctionPerson: HogFunctionType
let hogFunctionAll: HogFunctionType
let hogFunctionPersonAndEvent: HogFunctionType

beforeEach(() => {
hogFunctionPerson = createHogFunction({
Expand All @@ -127,6 +128,13 @@ describe('HogMasker', () => {
},
})

hogFunctionPersonAndEvent = createHogFunction({
masking: {
...HOG_MASK_EXAMPLES.personAndEvent.masking!,
ttl: 1,
},
})

hogFunctionAll = createHogFunction({
masking: {
...HOG_MASK_EXAMPLES.all.masking!,
Expand All @@ -145,20 +153,33 @@ describe('HogMasker', () => {
})

it('should mask with custom hog hash', async () => {
const globalsPerson1 = createHogExecutionGlobals({ person: { uuid: '1' } as any })
const globalsPerson2 = createHogExecutionGlobals({ person: { uuid: '2' } as any })
const globals1 = createHogExecutionGlobals({
person: { id: '1' } as any,
event: { event: '$pageview' } as any,
})
const globals2 = createHogExecutionGlobals({
person: { id: '2' } as any,
event: { event: '$autocapture' } as any,
})
const globals3 = createHogExecutionGlobals({
person: { id: '2' } as any,
event: { event: '$pageview' } as any,
})

const invocations = [
createInvocation(hogFunctionPerson, globalsPerson1),
createInvocation(hogFunctionAll, globalsPerson1),
createInvocation(hogFunctionPerson, globalsPerson2),
createInvocation(hogFunctionAll, globalsPerson2),
createInvocation(hogFunctionPerson, globals1),
createInvocation(hogFunctionAll, globals1),
createInvocation(hogFunctionPersonAndEvent, globals1),
createInvocation(hogFunctionPerson, globals2),
createInvocation(hogFunctionAll, globals2),
createInvocation(hogFunctionPersonAndEvent, globals2),
createInvocation(hogFunctionPersonAndEvent, globals3),
]
const res = await masker.filterByMasking(invocations)
expect(res.masked.length).toEqual(1)
expect(res.notMasked.length).toEqual(3)
expect(res.notMasked.length).toEqual(6)
const res2 = await masker.filterByMasking(invocations)
expect(res2.masked.length).toEqual(4)
expect(res2.masked.length).toEqual(7)
expect(res2.notMasked.length).toEqual(0)
})

Expand Down
178 changes: 89 additions & 89 deletions posthog/api/test/__snapshots__/test_api_docs.ambr
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion posthog/api/test/test_api_docs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@ def test_api_docs_generation_warnings_snapshot(self) -> None:

# we log lots of warnings when generating the schema
warnings = self._capsys.readouterr().err.split("\n")
assert warnings == self._snapshot
assert sorted(warnings) == self._snapshot
8 changes: 8 additions & 0 deletions posthog/clickhouse/migrations/0083_recreate_sessions_v1_after_limiting_teams.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
from posthog.models.sessions.sql import DROP_SESSION_MATERIALIZED_VIEW_SQL, SESSIONS_TABLE_MV_SQL

operations = [
# drop the mv, and recreate it with the new part of the WHERE clause
run_sql_with_exceptions(DROP_SESSION_MATERIALIZED_VIEW_SQL()),
run_sql_with_exceptions(SESSIONS_TABLE_MV_SQL()),
]
2 changes: 1 addition & 1 deletion posthog/clickhouse/test/__snapshots__/test_schema.ambr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2160,7 +2160,7 @@
sumIf(1, event='$autocapture') as autocapture_count

FROM posthog_test.sharded_events
WHERE `$session_id` IS NOT NULL AND `$session_id` != ''
WHERE `$session_id` IS NOT NULL AND `$session_id` != '' AND team_id IN (1, 2, 13610, 19279, 21173, 29929, 32050, 9910, 11775, 21129, 31490)
GROUP BY `$session_id`, team_id


Expand Down
53 changes: 30 additions & 23 deletions posthog/clickhouse/test/test_sessions_model.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
from posthog.clickhouse.client import sync_execute, query_with_columns
from posthog.models import Team
from posthog.test.base import (
_create_event,
ClickhouseTestMixin,
BaseTest,
ClickhouseDestroyTablesMixin,
)

distinct_id_counter = 0
Expand All @@ -21,7 +23,12 @@ def create_session_id():
return f"s{session_id_counter}"


class TestSessionsModel(ClickhouseTestMixin, BaseTest):
# only certain team ids can insert events into this legacy sessions table, see sessions/sql.py for more info
TEAM_ID = 2
TEAM = Team(id=TEAM_ID)


class TestSessionsModel(ClickhouseDestroyTablesMixin, ClickhouseTestMixin, BaseTest):
def select_by_session_id(self, session_id):
return query_with_columns(
"""
Expand All @@ -34,15 +41,15 @@ def select_by_session_id(self, session_id):
""",
{
"session_id": session_id,
"team_id": self.team.id,
"team_id": TEAM_ID,
},
)

def test_it_creates_session_when_creating_event(self):
distinct_id = create_distinct_id()
session_id = create_session_id()
_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$current_url": "/", "$session_id": session_id},
Expand All @@ -60,7 +67,7 @@ def test_it_creates_session_when_creating_event(self):
""",
{
"distinct_id": distinct_id,
"team_id": self.team.id,
"team_id": TEAM_ID,
},
)

Expand All @@ -72,14 +79,14 @@ def test_handles_different_distinct_id_across_same_session(self):
session_id = create_session_id()

_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id1,
properties={"$session_id": session_id},
timestamp="2024-03-08",
)
_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id2,
properties={"$session_id": session_id},
Expand All @@ -96,28 +103,28 @@ def test_handles_entry_and_exit_urls(self):
session_id = create_session_id()

_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$current_url": "/entry", "$session_id": session_id},
timestamp="2024-03-08:01",
)
_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$current_url": "/middle", "$session_id": session_id},
timestamp="2024-03-08:02",
)
_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$current_url": "/middle", "$session_id": session_id},
timestamp="2024-03-08:03",
)
_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$current_url": "/exit", "$session_id": session_id},
Expand All @@ -136,14 +143,14 @@ def test_handles_initial_utm_properties(self):
session_id = create_session_id()

_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$session_id": session_id, "utm_source": "source"},
timestamp="2024-03-08",
)
_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$session_id": session_id, "utm_source": "other_source"},
Expand All @@ -159,35 +166,35 @@ def test_counts_pageviews_autocaptures_and_events(self):
session_id = create_session_id()

_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$session_id": session_id},
timestamp="2024-03-08",
)
_create_event(
team=self.team,
team=TEAM,
event="$autocapture",
distinct_id=distinct_id,
properties={"$session_id": session_id},
timestamp="2024-03-08",
)
_create_event(
team=self.team,
team=TEAM,
event="$autocapture",
distinct_id=distinct_id,
properties={"$session_id": session_id},
timestamp="2024-03-08",
)
_create_event(
team=self.team,
team=TEAM,
event="other event",
distinct_id=distinct_id,
properties={"$session_id": session_id},
timestamp="2024-03-08",
)
_create_event(
team=self.team,
team=TEAM,
event="$pageleave",
distinct_id=distinct_id,
properties={"$session_id": session_id},
Expand All @@ -209,14 +216,14 @@ def test_separates_sessions_across_same_user(self):
session_id3 = create_session_id()

_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$session_id": session_id1},
timestamp="2024-03-08",
)
_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$session_id": session_id2},
Expand All @@ -235,7 +242,7 @@ def test_select_from_sessions(self):
distinct_id = create_distinct_id()
session_id = create_session_id()
_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$session_id": session_id},
Expand All @@ -260,7 +267,7 @@ def test_select_from_sessions(self):
""",
{
"session_id": session_id,
"team_id": self.team.id,
"team_id": TEAM_ID,
},
)
self.assertEqual(len(responses), 1)
Expand All @@ -270,7 +277,7 @@ def test_select_from_sessions_mv(self):
distinct_id = create_distinct_id()
session_id = create_session_id()
_create_event(
team=self.team,
team=TEAM,
event="$pageview",
distinct_id=distinct_id,
properties={"$session_id": session_id},
Expand All @@ -295,7 +302,7 @@ def test_select_from_sessions_mv(self):
""",
{
"session_id": session_id,
"team_id": self.team.id,
"team_id": TEAM_ID,
},
)
self.assertEqual(len(responses), 1)
Loading

0 comments on commit 849b918

Please sign in to comment.