Skip to content

Commit

Permalink
chore(experiments): Add assertions for property math operations (#26952)
Browse files Browse the repository at this point in the history 
Co-authored-by: Juraj Majerik <[email protected]>
  • Loading branch information
@danielbachhuber @jurajmajerik
danielbachhuber and jurajmajerik authored Dec 17, 2024
1 parent 9cbcf09 commit 22a9101
Showing 1 changed file with 215 additions and 0 deletions.
215 changes: 215 additions & 0 deletions posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1522,6 +1522,221 @@ def test_query_runner_standard_flow_v2_stats(self):
self.assertEqual(test_variant.count, 5.0)
self.assertEqual(test_variant.exposure, 1.0)

@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner_property_math_sum(self):
self._test_query_runner_property_math(
math="sum",
expected_control={
"count": 10,
"absolute_exposure": 5,
"data": [0.0, 0.0, 1.0, 3.0, 6.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0],
},
expected_test={
"count": 90,
"absolute_exposure": 10,
"data": [0.0, 0.0, 2.0, 6.0, 12.0, 20.0, 30.0, 42.0, 56.0, 72.0, 90.0, 90.0, 90.0, 90.0, 90.0],
},
)

@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner_property_math_avg(self):
self._test_query_runner_property_math(
math="avg",
expected_control={
"count": 10,
"absolute_exposure": 5,
"data": [0.0, 0.0, 1.0, 3.0, 6.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0],
},
expected_test={
"count": 90,
"absolute_exposure": 10,
"data": [0.0, 0.0, 2.0, 6.0, 12.0, 20.0, 30.0, 42.0, 56.0, 72.0, 90.0, 90.0, 90.0, 90.0, 90.0],
},
)

@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner_property_math_min(self):
self._test_query_runner_property_math(
math="min",
expected_control={
"count": 5,
"absolute_exposure": 5,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
},
expected_test={
"count": 10,
"absolute_exposure": 10,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0],
},
)

@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner_property_math_max(self):
self._test_query_runner_property_math(
math="max",
expected_control={
"count": 5,
"absolute_exposure": 5,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
},
expected_test={
"count": 10,
"absolute_exposure": 10,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0],
},
)

@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner_property_math_median(self):
self._test_query_runner_property_math(
math="median",
expected_control={
"count": 5,
"absolute_exposure": 5,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
},
expected_test={
"count": 10,
"absolute_exposure": 10,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0],
},
)

@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner_property_math_p90(self):
self._test_query_runner_property_math(
math="p90",
expected_control={
"count": 5,
"absolute_exposure": 5,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
},
expected_test={
"count": 10,
"absolute_exposure": 10,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0],
},
)

@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner_property_math_p95(self):
self._test_query_runner_property_math(
math="p95",
expected_control={
"count": 5,
"absolute_exposure": 5,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
},
expected_test={
"count": 10,
"absolute_exposure": 10,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0],
},
)

@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner_property_math_p99(self):
self._test_query_runner_property_math(
math="p99",
expected_control={
"count": 5,
"absolute_exposure": 5,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
},
expected_test={
"count": 10,
"absolute_exposure": 10,
"data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0],
},
)

def _test_query_runner_property_math(self, math, expected_control, expected_test):
feature_flag = self.create_feature_flag()
experiment = self.create_experiment(feature_flag=feature_flag, start_date=datetime(2020, 1, 1))

feature_flag_property = f"$feature/{feature_flag.key}"

# control values are 0, 1, 2, 3, 4
# test values are 0, 2, 4, 6, 8, 10, 12, 14, 16, 18

# Populate metric + exposure events
for variant, count in [("control", 5), ("test", 10)]:
for i in range(count):
_create_event(
team=self.team,
event="$feature_flag_called",
distinct_id=f"user_{variant}_{i}",
properties={
"$feature_flag_response": variant,
feature_flag_property: variant,
"$feature_flag": feature_flag.key,
},
timestamp=datetime(2020, 1, i + 1),
)
_create_event(
team=self.team,
event="purchase",
distinct_id=f"user_{variant}_{i}",
properties={
feature_flag_property: variant,
"amount": i * (1 if variant == "control" else 2),
},
timestamp=datetime(2020, 1, i + 2),
)

count_query = TrendsQuery(
series=[
EventsNode(
event="purchase",
math=math,
math_property="amount",
math_property_type="event_properties",
)
]
)
exposure_query = TrendsQuery(series=[EventsNode(event="$feature_flag_called")])
experiment_query = ExperimentTrendsQuery(
experiment_id=experiment.id,
kind="ExperimentTrendsQuery",
count_query=count_query,
exposure_query=exposure_query,
)

experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}]
experiment.save()

query_runner = ExperimentTrendsQueryRunner(
query=ExperimentTrendsQuery(**experiment.metrics[0]["query"]), team=self.team
)

flush_persons_and_events()

result = query_runner.calculate()

trend_result = cast(ExperimentTrendsQueryResponse, result)

self.assertEqual(len(result.variants), 2)

control_result = next(variant for variant in trend_result.variants if variant.key == "control")
test_result = next(variant for variant in trend_result.variants if variant.key == "test")

control_insight = next(variant for variant in trend_result.insight if variant["breakdown_value"] == "control")
test_insight = next(variant for variant in trend_result.insight if variant["breakdown_value"] == "test")

self.assertEqual(control_result.count, expected_control["count"])
self.assertEqual(test_result.count, expected_test["count"])
self.assertEqual(control_result.absolute_exposure, expected_control["absolute_exposure"])
self.assertEqual(test_result.absolute_exposure, expected_test["absolute_exposure"])

self.assertEqual(
control_insight["data"],
expected_control["data"],
)
self.assertEqual(
test_insight["data"],
expected_test["data"],
)

@freeze_time("2020-01-01T12:00:00Z")
def test_validate_event_variants_no_events(self):
feature_flag = self.create_feature_flag()
Expand Down

0 comments on commit 22a9101

Please sign in to comment.