diff --git a/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py index bba1366c33418..9ed6636bc2324 100644 --- a/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py +++ b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py @@ -1522,6 +1522,221 @@ def test_query_runner_standard_flow_v2_stats(self): self.assertEqual(test_variant.count, 5.0) self.assertEqual(test_variant.exposure, 1.0) + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner_property_math_sum(self): + self._test_query_runner_property_math( + math="sum", + expected_control={ + "count": 10, + "absolute_exposure": 5, + "data": [0.0, 0.0, 1.0, 3.0, 6.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0], + }, + expected_test={ + "count": 90, + "absolute_exposure": 10, + "data": [0.0, 0.0, 2.0, 6.0, 12.0, 20.0, 30.0, 42.0, 56.0, 72.0, 90.0, 90.0, 90.0, 90.0, 90.0], + }, + ) + + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner_property_math_avg(self): + self._test_query_runner_property_math( + math="avg", + expected_control={ + "count": 10, + "absolute_exposure": 5, + "data": [0.0, 0.0, 1.0, 3.0, 6.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0], + }, + expected_test={ + "count": 90, + "absolute_exposure": 10, + "data": [0.0, 0.0, 2.0, 6.0, 12.0, 20.0, 30.0, 42.0, 56.0, 72.0, 90.0, 90.0, 90.0, 90.0, 90.0], + }, + ) + + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner_property_math_min(self): + self._test_query_runner_property_math( + math="min", + expected_control={ + "count": 5, + "absolute_exposure": 5, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], + }, + expected_test={ + "count": 10, + "absolute_exposure": 10, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0], + }, + ) + + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner_property_math_max(self): + self._test_query_runner_property_math( + math="max", + expected_control={ + "count": 5, + "absolute_exposure": 5, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], + }, + expected_test={ + "count": 10, + "absolute_exposure": 10, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0], + }, + ) + + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner_property_math_median(self): + self._test_query_runner_property_math( + math="median", + expected_control={ + "count": 5, + "absolute_exposure": 5, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], + }, + expected_test={ + "count": 10, + "absolute_exposure": 10, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0], + }, + ) + + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner_property_math_p90(self): + self._test_query_runner_property_math( + math="p90", + expected_control={ + "count": 5, + "absolute_exposure": 5, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], + }, + expected_test={ + "count": 10, + "absolute_exposure": 10, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0], + }, + ) + + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner_property_math_p95(self): + self._test_query_runner_property_math( + math="p95", + expected_control={ + "count": 5, + "absolute_exposure": 5, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], + }, + expected_test={ + "count": 10, + "absolute_exposure": 10, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0], + }, + ) + + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner_property_math_p99(self): + self._test_query_runner_property_math( + math="p99", + expected_control={ + "count": 5, + "absolute_exposure": 5, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], + }, + expected_test={ + "count": 10, + "absolute_exposure": 10, + "data": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 10.0, 10.0, 10.0, 10.0], + }, + ) + + def _test_query_runner_property_math(self, math, expected_control, expected_test): + feature_flag = self.create_feature_flag() + experiment = self.create_experiment(feature_flag=feature_flag, start_date=datetime(2020, 1, 1)) + + feature_flag_property = f"$feature/{feature_flag.key}" + + # control values are 0, 1, 2, 3, 4 + # test values are 0, 2, 4, 6, 8, 10, 12, 14, 16, 18 + + # Populate metric + exposure events + for variant, count in [("control", 5), ("test", 10)]: + for i in range(count): + _create_event( + team=self.team, + event="$feature_flag_called", + distinct_id=f"user_{variant}_{i}", + properties={ + "$feature_flag_response": variant, + feature_flag_property: variant, + "$feature_flag": feature_flag.key, + }, + timestamp=datetime(2020, 1, i + 1), + ) + _create_event( + team=self.team, + event="purchase", + distinct_id=f"user_{variant}_{i}", + properties={ + feature_flag_property: variant, + "amount": i * (1 if variant == "control" else 2), + }, + timestamp=datetime(2020, 1, i + 2), + ) + + count_query = TrendsQuery( + series=[ + EventsNode( + event="purchase", + math=math, + math_property="amount", + math_property_type="event_properties", + ) + ] + ) + exposure_query = TrendsQuery(series=[EventsNode(event="$feature_flag_called")]) + experiment_query = ExperimentTrendsQuery( + experiment_id=experiment.id, + kind="ExperimentTrendsQuery", + count_query=count_query, + exposure_query=exposure_query, + ) + + experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}] + experiment.save() + + query_runner = ExperimentTrendsQueryRunner( + query=ExperimentTrendsQuery(**experiment.metrics[0]["query"]), team=self.team + ) + + flush_persons_and_events() + + result = query_runner.calculate() + + trend_result = cast(ExperimentTrendsQueryResponse, result) + + self.assertEqual(len(result.variants), 2) + + control_result = next(variant for variant in trend_result.variants if variant.key == "control") + test_result = next(variant for variant in trend_result.variants if variant.key == "test") + + control_insight = next(variant for variant in trend_result.insight if variant["breakdown_value"] == "control") + test_insight = next(variant for variant in trend_result.insight if variant["breakdown_value"] == "test") + + self.assertEqual(control_result.count, expected_control["count"]) + self.assertEqual(test_result.count, expected_test["count"]) + self.assertEqual(control_result.absolute_exposure, expected_control["absolute_exposure"]) + self.assertEqual(test_result.absolute_exposure, expected_test["absolute_exposure"]) + + self.assertEqual( + control_insight["data"], + expected_control["data"], + ) + self.assertEqual( + test_insight["data"], + expected_test["data"], + ) + @freeze_time("2020-01-01T12:00:00Z") def test_validate_event_variants_no_events(self): feature_flag = self.create_feature_flag()