feat(experiments): holdout groups UI

ee/clickhouse/queries/experiments/funnel_experiment_result.py

@@ -13,6 +13,7 @@
     calculate_credible_intervals,
     calculate_probabilities,
 )
+from posthog.models.experiment import ExperimentHoldout
 from posthog.models.feature_flag import FeatureFlag
 from posthog.models.filters.filter import Filter
 from posthog.models.team import Team
@@ -54,10 +55,13 @@ def __init__(
         feature_flag: FeatureFlag,
         experiment_start_date: datetime,
         experiment_end_date: Optional[datetime] = None,
+        holdout: Optional[ExperimentHoldout] = None,
         funnel_class: type[ClickhouseFunnel] = ClickhouseFunnel,
     ):
         breakdown_key = f"$feature/{feature_flag.key}"
         self.variants = [variant["key"] for variant in feature_flag.variants]
+        if holdout:
+            self.variants.append(f"holdout-{holdout.id}")
 
         # our filters assume that the given time ranges are in the project timezone.
         # while start and end date are in UTC.

ee/clickhouse/queries/experiments/trend_experiment_result.py

@@ -22,6 +22,7 @@
     calculate_credible_intervals,
     calculate_probabilities,
 )
+from posthog.models.experiment import ExperimentHoldout
 from posthog.models.feature_flag import FeatureFlag
 from posthog.models.filters.filter import Filter
 from posthog.models.team import Team
@@ -81,9 +82,12 @@ def __init__(
         experiment_end_date: Optional[datetime] = None,
         trend_class: type[Trends] = Trends,
         custom_exposure_filter: Optional[Filter] = None,
+        holdout: Optional[ExperimentHoldout] = None,
     ):
         breakdown_key = f"$feature/{feature_flag.key}"
         self.variants = [variant["key"] for variant in feature_flag.variants]
+        if holdout:
+            self.variants.append(f"holdout-{holdout.id}")
 
         # our filters assume that the given time ranges are in the project timezone.
         # while start and end date are in UTC.

ee/clickhouse/views/experiments.py

@@ -50,6 +50,7 @@ def _calculate_experiment_results(experiment: Experiment, refresh: bool = False)
             experiment.feature_flag,
             experiment.start_date,
             experiment.end_date,
+            holdout=experiment.holdout,
             custom_exposure_filter=exposure_filter,
         ).get_results()
     else:
@@ -59,6 +60,7 @@ def _calculate_experiment_results(experiment: Experiment, refresh: bool = False)
             experiment.feature_flag,
             experiment.start_date,
             experiment.end_date,
+            holdout=experiment.holdout,
         ).get_results()
 
     return _experiment_results_cached(

frontend/src/lib/constants.tsx

@@ -220,6 +220,7 @@ export const FEATURE_FLAGS = {
     LEGACY_ACTION_WEBHOOKS: 'legacy-action-webhooks', // owner: @mariusandra #team-cdp
     SESSION_REPLAY_URL_TRIGGER: 'session-replay-url-trigger', // owner: @richard-better #team-replay
     REPLAY_TEMPLATES: 'replay-templates', // owner: @raquelmsmith #team-replay
+    EXPERIMENTS_HOLDOUTS: 'experiments-holdouts', // owner: @jurajmajerik #team-experiments
 } as const
 export type FeatureFlagKey = (typeof FEATURE_FLAGS)[keyof typeof FEATURE_FLAGS]
 

frontend/src/scenes/experiments/ExperimentForm.tsx

@@ -5,11 +5,12 @@ import { LemonDivider, LemonInput, LemonTextArea, Tooltip } from '@posthog/lemon
 import { BindLogic, useActions, useValues } from 'kea'
 import { Form, Group } from 'kea-forms'
 import { ExperimentVariantNumber } from 'lib/components/SeriesGlyph'
-import { MAX_EXPERIMENT_VARIANTS } from 'lib/constants'
+import { FEATURE_FLAGS, MAX_EXPERIMENT_VARIANTS } from 'lib/constants'
 import { IconChevronLeft } from 'lib/lemon-ui/icons'
 import { LemonButton } from 'lib/lemon-ui/LemonButton'
 import { LemonField } from 'lib/lemon-ui/LemonField'
 import { LemonRadio } from 'lib/lemon-ui/LemonRadio'
+import { LemonSelect } from 'lib/lemon-ui/LemonSelect'
 import { capitalizeFirstLetter } from 'lib/utils'
 import { useEffect } from 'react'
 import { insightDataLogic } from 'scenes/insights/insightDataLogic'
@@ -23,7 +24,7 @@ import { experimentLogic } from './experimentLogic'
 import { ExperimentInsightCreator } from './MetricSelector'
 
 const StepInfo = (): JSX.Element => {
-    const { experiment } = useValues(experimentLogic)
+    const { experiment, featureFlags } = useValues(experimentLogic)
     const { addExperimentGroup, removeExperimentGroup, moveToNextFormStep } = useActions(experimentLogic)
 
     return (
@@ -134,6 +135,14 @@ const StepInfo = (): JSX.Element => {
                         </div>
                     </div>
                 </div>
+                {featureFlags[FEATURE_FLAGS.EXPERIMENTS_HOLDOUTS] && (
+                    <div>
+                        <h3>Holdout group</h3>
+                        <div className="text-xs text-muted">Exclude a stable group of users from the experiment.</div>
+                        <LemonDivider />
+                        <HoldoutSelector />
+                    </div>
+                )}
             </div>
             <LemonButton
                 className="mt-2"
@@ -276,6 +285,33 @@ const StepGoal = (): JSX.Element => {
     )
 }
 
+export const HoldoutSelector = (): JSX.Element => {
+    const { experiment, holdouts } = useValues(experimentLogic)
+    const { setExperiment } = useActions(experimentLogic)
+
+    const holdoutOptions = holdouts.map((holdout) => ({
+        value: holdout.id,
+        label: holdout.name,
+    }))
+    holdoutOptions.unshift({ value: null, label: 'No holdout' })
+
+    return (
+        <div className="mt-4 mb-8">
+            <LemonSelect
+                options={holdoutOptions}
+                value={experiment.holdout || null}
+                onChange={(value) => {
+                    setExperiment({
+                        ...experiment,
+                        holdout: value,
+                    })
+                }}
+                data-attr="experiment-holdout-selector"
+            />
+        </div>
+    )
+}
+
 export function ExperimentForm(): JSX.Element {
     const { currentFormStep, props } = useValues(experimentLogic)
     const { setCurrentFormStep } = useActions(experimentLogic)

frontend/src/scenes/experiments/ExperimentView/ExperimentView.tsx

@@ -2,6 +2,8 @@ import '../Experiment.scss'
 
 import { LemonDivider } from '@posthog/lemon-ui'
 import { useActions, useValues } from 'kea'
+import { FEATURE_FLAGS } from 'lib/constants'
+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
 
 import { ExperimentImplementationDetails } from '../ExperimentImplementationDetails'
 import { experimentLogic } from '../experimentLogic'
@@ -15,6 +17,7 @@ import {
 import { DataCollection } from './DataCollection'
 import { DistributionTable } from './DistributionTable'
 import { ExperimentExposureModal, ExperimentGoalModal, Goal } from './Goal'
+import { HoldoutSelector } from './HoldoutSelector'
 import { Info } from './Info'
 import { Overview } from './Overview'
 import { ReleaseConditionsTable } from './ReleaseConditionsTable'
@@ -24,6 +27,7 @@ import { SecondaryMetricsTable } from './SecondaryMetricsTable'
 export function ExperimentView(): JSX.Element {
     const { experiment, experimentLoading, experimentResultsLoading, experimentId, experimentResults } =
         useValues(experimentLogic)
+    const { featureFlags } = useValues(featureFlagLogic)
 
     const { updateExperimentSecondaryMetrics } = useActions(experimentLogic)
 
@@ -47,6 +51,7 @@ export function ExperimentView(): JSX.Element {
                                 <div className="xl:flex">
                                     <div className="w-1/2 pr-2">
                                         <Goal />
+                                        {featureFlags[FEATURE_FLAGS.EXPERIMENTS_HOLDOUTS] && <HoldoutSelector />}
                                     </div>
 
                                     <div className="w-1/2 xl:pl-2 mt-8 xl:mt-0">
@@ -60,6 +65,7 @@ export function ExperimentView(): JSX.Element {
                                 <div className="xl:flex">
                                     <div className="w-1/2 pr-2">
                                         <Goal />
+                                        {featureFlags[FEATURE_FLAGS.EXPERIMENTS_HOLDOUTS] && <HoldoutSelector />}
                                     </div>
 
                                     <div className="w-1/2 xl:pl-2 mt-8 xl:mt-0">

frontend/src/scenes/experiments/ExperimentView/HoldoutSelector.tsx

@@ -0,0 +1,47 @@
+import { IconInfo } from '@posthog/icons'
+import { LemonSelect, Tooltip } from '@posthog/lemon-ui'
+import { useActions, useValues } from 'kea'
+
+import { experimentLogic } from '../experimentLogic'
+
+export function HoldoutSelector(): JSX.Element {
+    const { experiment, holdouts, isExperimentRunning } = useValues(experimentLogic)
+    const { setExperiment, updateExperiment } = useActions(experimentLogic)
+
+    const holdoutOptions = holdouts.map((holdout) => ({
+        value: holdout.id,
+        label: holdout.name,
+    }))
+    holdoutOptions.unshift({ value: null, label: 'No holdout' })
+
+    return (
+        <div className="mt-3">
+            <div className="inline-flex space-x-1">
+                <h4 className="font-semibold mb-0">Holdout group</h4>
+                <Tooltip title="Exclude a stable group of users from the experiment. This cannot be changed once the experiment is launched.">
+                    <IconInfo className="text-muted-alt text-base" />
+                </Tooltip>
+            </div>
+            <div className="mt-1">
+                <LemonSelect
+                    disabledReason={
+                        isExperimentRunning &&
+                        !experiment.end_date &&
+                        'The holdout group cannot be changed once the experiment is launched.'
+                    }
+                    size="xsmall"
+                    options={holdoutOptions}
+                    value={experiment.holdout || null}
+                    onChange={(value) => {
+                        setExperiment({
+                            ...experiment,
+                            holdout: value,
+                        })
+                        updateExperiment({ holdout: value })
+                    }}
+                    data-attr="experiment-holdout-selector"
+                />
+            </div>
+        </div>
+    )
+}

frontend/src/scenes/experiments/ExperimentView/components.tsx

@@ -52,8 +52,23 @@ export function VariantTag({
 }): JSX.Element {
     const { experimentResults, getIndexForVariant } = useValues(experimentLogic({ experimentId }))
 
+    if (variantKey.startsWith('holdout-')) {
+        return (
+            <span className="flex items-center space-x-2">
+                <div
+                    className="w-2 h-2 rounded-full mr-0.5"
+                    // eslint-disable-next-line react/forbid-dom-props
+                    style={{
+                        backgroundColor: getExperimentInsightColour(getIndexForVariant(experimentResults, variantKey)),
+                    }}
+                />
+                <LemonTag type="option">{variantKey}</LemonTag>
+            </span>
+        )
+    }
+
     return (
-        <span className="flex items-center space-x-1">
+        <span className="flex items-center space-x-2">
             <div
                 className="w-2 h-2 rounded-full mr-0.5"
                 // eslint-disable-next-line react/forbid-dom-props