feat(experiments): validate events for Trend experiments (#20058)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

ee/clickhouse/queries/experiments/trend_experiment_result.py

@@ -80,7 +80,7 @@ def __init__(
         custom_exposure_filter: Optional[Filter] = None,
     ):
         breakdown_key = f"$feature/{feature_flag.key}"
-        variants = [variant["key"] for variant in feature_flag.variants]
+        self.variants = [variant["key"] for variant in feature_flag.variants]
 
         # our filters assume that the given time ranges are in the project timezone.
         # while start and end date are in UTC.
@@ -104,7 +104,7 @@ def __init__(
                 "properties": [
                     {
                         "key": breakdown_key,
-                        "value": variants,
+                        "value": self.variants,
                         "operator": "exact",
                         "type": "event",
                     }
@@ -158,7 +158,7 @@ def __init__(
                         "properties": [
                             {
                                 "key": breakdown_key,
-                                "value": variants,
+                                "value": self.variants,
                                 "operator": "exact",
                                 "type": "event",
                             }
@@ -187,7 +187,7 @@ def __init__(
                         "properties": [
                             {
                                 "key": "$feature_flag_response",
-                                "value": variants,
+                                "value": self.variants,
                                 "operator": "exact",
                                 "type": "event",
                             },
@@ -209,6 +209,9 @@ def __init__(
     def get_results(self):
         insight_results = self.insight.run(self.query_filter, self.team)
         exposure_results = self.insight.run(self.exposure_filter, self.team)
+
+        validate_event_variants(insight_results, self.variants)
+
         control_variant, test_variants = self.get_variants(insight_results, exposure_results)
 
         probabilities = self.calculate_results(control_variant, test_variants)
@@ -437,3 +440,36 @@ def calculate_p_value(control_variant: Variant, test_variants: List[Variant]) ->
         best_test_variant.count,
         best_test_variant.exposure,
     )
+
+
+def validate_event_variants(insight_results, variants):
+    if not insight_results or not insight_results[0]:
+        raise ValidationError("No experiment events have been ingested yet.", code="no-events")
+
+    missing_variants = []
+
+    # Check if "control" is present
+    control_found = False
+    for event in insight_results:
+        event_variant = event.get("breakdown_value")
+        if event_variant == "control":
+            control_found = True
+            break
+    if not control_found:
+        missing_variants.append("control")
+
+    # Check if at least one of the test variants is present
+    test_variants = [variant for variant in variants if variant != "control"]
+    test_variant_found = False
+    for event in insight_results:
+        event_variant = event.get("breakdown_value")
+        if event_variant in test_variants:
+            test_variant_found = True
+            break
+    if not test_variant_found:
+        missing_variants.extend(test_variants)
+
+    if not len(missing_variants) == 0:
+        missing_variants_str = ", ".join(missing_variants)
+        message = f"No experiment events have been ingested yet for the following variants: {missing_variants_str}"
+        raise ValidationError(message, code=f"missing-flag-variants::{missing_variants_str}")

ee/clickhouse/queries/test/test_experiments.py

@@ -1,13 +1,18 @@
 import unittest
-from ee.clickhouse.queries.experiments.funnel_experiment_result import validate_event_variants
+from ee.clickhouse.queries.experiments.funnel_experiment_result import (
+    validate_event_variants as validate_funnel_event_variants,
+)
+from ee.clickhouse.queries.experiments.trend_experiment_result import (
+    validate_event_variants as validate_trend_event_variants,
+)
 from rest_framework.exceptions import ValidationError
 
 
-class TestExperiments(unittest.TestCase):
+class TestFunnelExperiments(unittest.TestCase):
     def test_validate_event_variants_no_events(self):
         expected_code = "no-events"
         with self.assertRaises(ValidationError) as context:
-            validate_event_variants([], ["test", "control"])
+            validate_funnel_event_variants([], ["test", "control"])
 
         self.assertEqual(expected_code, context.exception.detail[0].code)
 
@@ -45,7 +50,7 @@ def test_validate_event_variants_missing_variants(self):
 
         expected_code = "missing-flag-variants::control"
         with self.assertRaises(ValidationError) as context:
-            validate_event_variants(funnel_results, ["test", "control"])
+            validate_funnel_event_variants(funnel_results, ["test", "control"])
 
         self.assertEqual(expected_code, context.exception.detail[0].code)
 
@@ -83,6 +88,65 @@ def test_validate_event_variants_ignore_old_variant(self):
 
         expected_code = "missing-flag-variants::control"
         with self.assertRaises(ValidationError) as context:
-            validate_event_variants(funnel_results, ["test", "control"])
+            validate_funnel_event_variants(funnel_results, ["test", "control"])
+
+        self.assertEqual(expected_code, context.exception.detail[0].code)
+
+
+class TestTrendExperiments(unittest.TestCase):
+    def test_validate_event_variants_no_events(self):
+        expected_code = "no-events"
+        with self.assertRaises(ValidationError) as context:
+            validate_trend_event_variants([], ["test", "control"])
+
+        self.assertEqual(expected_code, context.exception.detail[0].code)
+
+    def test_validate_event_variants_missing_variants(self):
+        insight_results = [
+            {
+                "action": {
+                    "id": "step-b-0",
+                    "type": "events",
+                    "order": 0,
+                    "name": "step-b-0",
+                },
+                "label": "test",
+                "breakdown_value": "test",
+            }
+        ]
+
+        expected_code = "missing-flag-variants::control"
+        with self.assertRaises(ValidationError) as context:
+            validate_trend_event_variants(insight_results, ["test", "control"])
+
+        self.assertEqual(expected_code, context.exception.detail[0].code)
+
+    def test_validate_event_variants_ignore_old_variant(self):
+        insight_results = [
+            {
+                "action": {
+                    "id": "step-b-0",
+                    "type": "events",
+                    "order": 0,
+                    "name": "step-b-0",
+                },
+                "label": "test",
+                "breakdown_value": "test",
+            },
+            {
+                "action": {
+                    "id": "step-b-0",
+                    "type": "events",
+                    "order": 0,
+                    "name": "step-b-0",
+                },
+                "label": "test",
+                "breakdown_value": "old-variant",
+            },
+        ]
+
+        expected_code = "missing-flag-variants::control"
+        with self.assertRaises(ValidationError) as context:
+            validate_trend_event_variants(insight_results, ["test", "control"])
 
         self.assertEqual(expected_code, context.exception.detail[0].code)