feat(experiments): Store stats version on Experiment model (#27146)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

ee/clickhouse/views/experiments.py

@@ -192,6 +192,7 @@ class Meta:
             "type",
             "metrics",
             "metrics_secondary",
+            "stats_config",
         ]
         read_only_fields = [
             "id",
@@ -376,6 +377,7 @@ def update(self, instance: Experiment, validated_data: dict, *args: Any, **kwarg
             "holdout",
             "metrics",
             "metrics_secondary",
+            "stats_config",
         }
         given_keys = set(validated_data.keys())
         extra_keys = given_keys - expected_keys

ee/clickhouse/views/test/test_clickhouse_experiments.py

@@ -102,6 +102,11 @@ def test_creating_updating_basic_experiment(self):
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
         self.assertEqual(response.json()["name"], "Test Experiment")
         self.assertEqual(response.json()["feature_flag_key"], ff_key)
+        self.assertEqual(response.json()["stats_config"], {})
+
+        id = response.json()["id"]
+        experiment = Experiment.objects.get(pk=id)
+        self.assertEqual(experiment.get_stats_config("version"), None)
 
         created_ff = FeatureFlag.objects.get(key=ff_key)
 
@@ -110,20 +115,20 @@ def test_creating_updating_basic_experiment(self):
         self.assertEqual(created_ff.filters["multivariate"]["variants"][1]["key"], "test")
         self.assertEqual(created_ff.filters["groups"][0]["properties"], [])
 
-        id = response.json()["id"]
         end_date = "2021-12-10T00:00"
 
         # Now update
         response = self.client.patch(
             f"/api/projects/{self.team.id}/experiments/{id}",
-            {"description": "Bazinga", "end_date": end_date},
+            {"description": "Bazinga", "end_date": end_date, "stats_config": {"version": 2}},
         )
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
 
         experiment = Experiment.objects.get(pk=id)
         self.assertEqual(experiment.description, "Bazinga")
         self.assertEqual(experiment.end_date.strftime("%Y-%m-%dT%H:%M"), end_date)
+        self.assertEqual(experiment.get_stats_config("version"), 2)
 
     def test_creating_updating_web_experiment(self):
         ff_key = "a-b-tests"

frontend/src/queries/schema.json

@@ -6017,9 +6017,6 @@
                 },
                 "response": {
                     "$ref": "#/definitions/ExperimentFunnelsQueryResponse"
-                },
-                "stats_version": {
-                    "type": "integer"
                 }
             },
             "required": ["funnels_query", "kind"],
@@ -6121,9 +6118,6 @@
                 },
                 "response": {
                     "$ref": "#/definitions/ExperimentTrendsQueryResponse"
-                },
-                "stats_version": {
-                    "type": "integer"
                 }
             },
             "required": ["count_query", "kind"],

frontend/src/queries/schema.ts

@@ -2052,7 +2052,6 @@ export interface ExperimentFunnelsQuery extends DataNode<ExperimentFunnelsQueryR
     name?: string
     experiment_id?: integer
     funnels_query: FunnelsQuery
-    stats_version?: integer
 }
 
 export interface ExperimentTrendsQuery extends DataNode<ExperimentTrendsQueryResponse> {
@@ -2063,7 +2062,6 @@ export interface ExperimentTrendsQuery extends DataNode<ExperimentTrendsQueryRes
     // Defaults to $feature_flag_called if not specified
     // https://github.com/PostHog/posthog/blob/master/posthog/hogql_queries/experiments/experiment_trends_query_runner.py
     exposure_query?: TrendsQuery
-    stats_version?: integer
 }
 
 /**

frontend/src/scenes/experiments/ExperimentView/Info.tsx

@@ -1,5 +1,5 @@
 import { IconWarning } from '@posthog/icons'
-import { Link, ProfilePicture, Tooltip } from '@posthog/lemon-ui'
+import { LemonButton, Link, ProfilePicture, Tooltip } from '@posthog/lemon-ui'
 import { useActions, useValues } from 'kea'
 import { CopyToClipboardInline } from 'lib/components/CopyToClipboard'
 import { EditableField } from 'lib/components/EditableField/EditableField'
@@ -16,14 +16,16 @@ import { ExperimentDates } from './ExperimentDates'
 
 export function Info(): JSX.Element {
     const { experiment, featureFlags } = useValues(experimentLogic)
-    const { updateExperiment } = useActions(experimentLogic)
+    const { updateExperiment, setExperimentStatsVersion } = useActions(experimentLogic)
 
     const { created_by } = experiment
 
     if (!experiment.feature_flag) {
         return <></>
     }
 
+    const currentStatsVersion = experiment.stats_config?.version || 1
+
     return (
         <div>
             <div className="flex">
@@ -72,6 +74,28 @@ export function Info(): JSX.Element {
                             </Link>
                         </div>
                     )}
+                    {featureFlags[FEATURE_FLAGS.EXPERIMENT_STATS_V2] && (
+                        <div className="block">
+                            <div className="text-xs font-semibold uppercase tracking-wide">
+                                <span>Stats Version</span>
+                            </div>
+                            <div className="flex gap-1">
+                                {[1, 2].map((version) => (
+                                    <LemonButton
+                                        key={version}
+                                        size="xsmall"
+                                        type="tertiary"
+                                        active={currentStatsVersion === version}
+                                        onClick={() => {
+                                            setExperimentStatsVersion(version)
+                                        }}
+                                    >
+                                        v{version}
+                                    </LemonButton>
+                                ))}
+                            </div>
+                        </div>
+                    )}
                 </div>
 
                 <div className="w-1/2 flex flex-col justify-end">

frontend/src/scenes/experiments/experimentLogic.tsx

@@ -191,6 +191,7 @@ export const experimentLogic = kea<experimentLogicType>([
         updateExperimentGoal: true,
         updateExperimentCollectionGoal: true,
         changeExperimentStartDate: (startDate: string) => ({ startDate }),
+        setExperimentStatsVersion: (version: number) => ({ version }),
         launchExperiment: true,
         endExperiment: true,
         addExperimentGroup: true,
@@ -695,6 +696,12 @@ export const experimentLogic = kea<experimentLogicType>([
             actions.updateExperiment({ start_date: startDate })
             values.experiment && eventUsageLogic.actions.reportExperimentStartDateChange(values.experiment, startDate)
         },
+        setExperimentStatsVersion: async ({ version }, breakpoint) => {
+            actions.updateExperiment({ stats_config: { version } })
+            await breakpoint(100)
+            actions.loadMetricResults(true)
+            actions.loadSecondaryMetricResults(true)
+        },
         endExperiment: async () => {
             const endDate = dayjs()
             actions.updateExperiment({ end_date: endDate.toISOString() })

frontend/src/types.ts

@@ -3332,6 +3332,9 @@ export interface Experiment {
     updated_at: string | null
     holdout_id?: number | null
     holdout?: Holdout
+    stats_config?: {
+        version?: number
+    }
 }
 
 export interface FunnelExperimentVariant {

posthog/api/test/__snapshots__/test_feature_flag.ambr

@@ -1572,7 +1572,8 @@
          "posthog_experiment"."type",
          "posthog_experiment"."variants",
          "posthog_experiment"."metrics",
-         "posthog_experiment"."metrics_secondary"
+         "posthog_experiment"."metrics_secondary",
+         "posthog_experiment"."stats_config"
   FROM "posthog_experiment"
   WHERE "posthog_experiment"."exposure_cohort_id" = 99999
   '''

posthog/api/test/__snapshots__/test_organization_feature_flag.ambr

@@ -1857,7 +1857,8 @@
          "posthog_experiment"."type",
          "posthog_experiment"."variants",
          "posthog_experiment"."metrics",
-         "posthog_experiment"."metrics_secondary"
+         "posthog_experiment"."metrics_secondary",
+         "posthog_experiment"."stats_config"
   FROM "posthog_experiment"
   WHERE "posthog_experiment"."feature_flag_id" = 99999
   '''

posthog/hogql_queries/experiments/experiment_funnels_query_runner.py

@@ -51,7 +51,7 @@ def __init__(self, *args, **kwargs):
         if self.experiment.holdout:
             self.variants.append(f"holdout-{self.experiment.holdout.id}")
 
-        self.stats_version = self.query.stats_version or 1
+        self.stats_version = self.experiment.get_stats_config("version") or 1
 
         self.prepared_funnels_query = self._prepare_funnel_query()
         self.funnels_query_runner = FunnelsQueryRunner(

posthog/hogql_queries/experiments/experiment_trends_query_runner.py

@@ -66,7 +66,7 @@ def __init__(self, *args, **kwargs):
             self.variants.append(f"holdout-{self.experiment.holdout.id}")
         self.breakdown_key = f"$feature/{self.feature_flag.key}"
 
-        self.stats_version = self.query.stats_version or 1
+        self.stats_version = self.experiment.get_stats_config("version") or 1
 
         self.prepared_count_query = self._prepare_count_query()
         self.prepared_exposure_query = self._prepare_exposure_query()

posthog/hogql_queries/experiments/test/test_experiment_funnels_query_runner.py

@@ -141,6 +141,8 @@ def test_query_runner(self):
     def test_query_runner_v2(self):
         feature_flag = self.create_feature_flag()
         experiment = self.create_experiment(feature_flag=feature_flag)
+        experiment.stats_config = {"version": 2}
+        experiment.save()
 
         feature_flag_property = f"$feature/{feature_flag.key}"
 
@@ -152,7 +154,6 @@ def test_query_runner_v2(self):
             experiment_id=experiment.id,
             kind="ExperimentFunnelsQuery",
             funnels_query=funnels_query,
-            stats_version=2,
         )
 
         experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}]

posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py

@@ -1287,6 +1287,8 @@ def test_query_runner_with_avg_math(self):
     def test_query_runner_with_avg_math_v2_stats(self):
         feature_flag = self.create_feature_flag()
         experiment = self.create_experiment(feature_flag=feature_flag)
+        experiment.stats_config = {"version": 2}
+        experiment.save()
 
         feature_flag_property = f"$feature/{feature_flag.key}"
 
@@ -1302,7 +1304,6 @@ def test_query_runner_with_avg_math_v2_stats(self):
             kind="ExperimentTrendsQuery",
             count_query=count_query,
             exposure_query=exposure_query,
-            stats_version=2,
         )
 
         experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}]
@@ -1522,6 +1523,8 @@ def test_query_runner_standard_flow(self):
     def test_query_runner_standard_flow_v2_stats(self):
         feature_flag = self.create_feature_flag()
         experiment = self.create_experiment(feature_flag=feature_flag)
+        experiment.stats_config = {"version": 2}
+        experiment.save()
 
         ff_property = f"$feature/{feature_flag.key}"
         count_query = TrendsQuery(series=[EventsNode(event="$pageview")])
@@ -1532,7 +1535,6 @@ def test_query_runner_standard_flow_v2_stats(self):
             kind="ExperimentTrendsQuery",
             count_query=count_query,
             exposure_query=exposure_query,
-            stats_version=2,
         )
 
         experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}]

posthog/migrations/0538_experiment_stats_config.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.15 on 2025-01-06 12:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("posthog", "0537_data_color_themes"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="experiment",
+            name="stats_config",
+            field=models.JSONField(blank=True, default=dict, null=True),
+        ),
+    ]

posthog/migrations/max_migration.txt

@@ -1 +1 @@
-0537_data_color_themes
+0538_experiment_stats_config

posthog/models/experiment.py

@@ -46,9 +46,14 @@ class ExperimentType(models.TextChoices):
         "ExperimentSavedMetric", blank=True, related_name="experiments", through="ExperimentToSavedMetric"
     )
 
+    stats_config = models.JSONField(default=dict, null=True, blank=True)
+
     def get_feature_flag_key(self):
         return self.feature_flag.key
 
+    def get_stats_config(self, key: str):
+        return self.stats_config.get(key) if self.stats_config else None
+
     @property
     def is_draft(self):
         return not self.start_date

posthog/schema.py

@@ -6606,7 +6606,6 @@ class ExperimentTrendsQuery(BaseModel):
     )
     name: Optional[str] = None
     response: Optional[ExperimentTrendsQueryResponse] = None
-    stats_version: Optional[int] = None
 
 
 class FunnelPathsFilter(BaseModel):
@@ -6723,7 +6722,6 @@ class ExperimentFunnelsQuery(BaseModel):
     )
     name: Optional[str] = None
     response: Optional[ExperimentFunnelsQueryResponse] = None
-    stats_version: Optional[int] = None
 
 
 class FunnelCorrelationQuery(BaseModel):

posthog/tasks/test/__snapshots__/test_process_scheduled_changes.ambr

@@ -557,7 +557,8 @@
          "posthog_experiment"."type",
          "posthog_experiment"."variants",
          "posthog_experiment"."metrics",
-         "posthog_experiment"."metrics_secondary"
+         "posthog_experiment"."metrics_secondary",
+         "posthog_experiment"."stats_config"
   FROM "posthog_experiment"
   WHERE "posthog_experiment"."feature_flag_id" = 99999
   '''
@@ -1186,7 +1187,8 @@
          "posthog_experiment"."type",
          "posthog_experiment"."variants",
          "posthog_experiment"."metrics",
-         "posthog_experiment"."metrics_secondary"
+         "posthog_experiment"."metrics_secondary",
+         "posthog_experiment"."stats_config"
   FROM "posthog_experiment"
   WHERE "posthog_experiment"."feature_flag_id" = 99999
   '''