diff --git a/ee/clickhouse/queries/experiments/funnel_experiment_result.py b/ee/clickhouse/queries/experiments/funnel_experiment_result.py index 4044bcfd6ac7a..db462977059a5 100644 --- a/ee/clickhouse/queries/experiments/funnel_experiment_result.py +++ b/ee/clickhouse/queries/experiments/funnel_experiment_result.py @@ -59,6 +59,7 @@ def __init__( ): breakdown_key = f"$feature/{feature_flag.key}" self.variants = [variant["key"] for variant in feature_flag.variants] + self.filter = filter # our filters assume that the given time ranges are in the project timezone. # while start and end date are in UTC. @@ -86,6 +87,9 @@ def __init__( def get_results(self): funnel_results = self.funnel.run() filtered_results = [result for result in funnel_results if result[0]["breakdown_value"][0] in self.variants] + + validate_event_variants(filtered_results, self.variants) + control_variant, test_variants = self.get_variants(filtered_results) probabilities = self.calculate_results(control_variant, test_variants) @@ -292,3 +296,27 @@ def calculate_probability_of_winning_for_each(variants: List[Variant]) -> List[P total_test_probabilities = sum(probabilities[1:]) return [max(0, 1 - total_test_probabilities), *probabilities[1:]] + + +def validate_event_variants(filtered_results, variants): + if not filtered_results or not filtered_results[0]: + raise ValidationError("No experiment events have been ingested yet") + + eventsWithOrderZero = [] + for eventArr in filtered_results: + for event in eventArr: + if event.get("order") == 0: + eventsWithOrderZero.append(event) + if len(eventsWithOrderZero) == 0: + raise ValidationError("No events for the first funnel step have been ingested yet") + + missing_variants = set(variants) + for event in eventsWithOrderZero: + event_variant = event.get("breakdown_value")[0] + if event_variant in missing_variants: + missing_variants.discard(event_variant) + + if not len(missing_variants) == 0: + missing_variants_str = ", ".join(missing_variants) + message = f"No experiment events have been ingested yet for the following variants: {missing_variants_str}" + raise ValidationError(message) diff --git a/ee/clickhouse/queries/test/test_experiments.py b/ee/clickhouse/queries/test/test_experiments.py new file mode 100644 index 0000000000000..4ab2bdb8cba06 --- /dev/null +++ b/ee/clickhouse/queries/test/test_experiments.py @@ -0,0 +1,62 @@ +import unittest +from ee.clickhouse.queries.experiments.funnel_experiment_result import validate_event_variants +from rest_framework.exceptions import ValidationError + + +class TestExperiments(unittest.TestCase): + def test_validate_event_variants(self): + expected_message = "No experiment events have been ingested yet" + with self.assertRaises(ValidationError) as context: + validate_event_variants([], ["test", "control"]) + + self.assertIn(expected_message, str(context.exception)) + + def test_validate_event_variants_2(self): + filtered_results = [ + [ + { + "action_id": "step-2", + "name": "step-2", + "custom_name": None, + "order": 1, + "people": [], + "count": 3, + "type": "events", + "average_conversion_time": 0.3333333333333333, + "median_conversion_time": 0.0, + "breakdown": ["control"], + "breakdown_value": ["control"], + } + ] + ] + + expected_message = "No events for the first funnel step have been ingested yet" + with self.assertRaises(ValidationError) as context: + validate_event_variants(filtered_results, ["test", "control"]) + + self.assertIn(expected_message, str(context.exception)) + + def test_validate_event_variants_3(self): + filtered_results = [ + [ + { + "action_id": "step-1", + "name": "step-1", + "custom_name": None, + "order": 0, + "people": [], + "count": 3, + "type": "events", + "average_conversion_time": 0.3333333333333333, + "median_conversion_time": 0.0, + "breakdown": ["control"], + "breakdown_value": ["control"], + } + ] + ] + + expected_message = "No experiment events have been ingested yet for the following variants: test" + with self.assertRaises(ValidationError) as context: + validate_event_variants(filtered_results, ["test", "control"]) + + self.assertIn(expected_message, str(context.exception)) diff --git a/frontend/src/scenes/experiments/ExperimentResult.tsx b/frontend/src/scenes/experiments/ExperimentResult.tsx index 8ba87a58b5f89..64e6ee8e10403 100644 --- a/frontend/src/scenes/experiments/ExperimentResult.tsx +++ b/frontend/src/scenes/experiments/ExperimentResult.tsx @@ -184,9 +184,13 @@ export function ExperimentResult(): JSX.Element {