diff --git a/frontend/src/scenes/experiments/ExperimentView/SecondaryMetricsTable.tsx b/frontend/src/scenes/experiments/ExperimentView/SecondaryMetricsTable.tsx index 86776c0259219..066144566b081 100644 --- a/frontend/src/scenes/experiments/ExperimentView/SecondaryMetricsTable.tsx +++ b/frontend/src/scenes/experiments/ExperimentView/SecondaryMetricsTable.tsx @@ -131,6 +131,7 @@ export function SecondaryMetricsTable({ countDataForVariant, exposureCountDataForVariant, conversionRateForVariant, + credibleIntervalForVariant, experimentMathAggregationForTrends, getHighestProbabilityVariant, } = useValues(experimentLogic({ experimentId })) @@ -223,6 +224,24 @@ export function SecondaryMetricsTable({ ) }, }, + { + title: 'Credible interval (95%)', + render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element { + if (item.variant === 'control') { + return Baseline + } + const credibleInterval = credibleIntervalForVariant(targetResults || null, item.variant) + if (!credibleInterval) { + return <>— + } + const [lowerBound, upperBound] = credibleInterval + return ( +
{`[${lowerBound > 0 ? '+' : ''}${lowerBound.toFixed( + 2 + )}%, ${upperBound > 0 ? '+' : ''}${upperBound.toFixed(2)}%]`}
+ ) + }, + }, { title: 'Win probability', render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element { @@ -255,6 +274,25 @@ export function SecondaryMetricsTable({ return
{`${conversionRate.toFixed(2)}%`}
}, }, + { + title: 'Credible interval (95%)', + render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element { + if (item.variant === 'control') { + return Baseline + } + + const credibleInterval = credibleIntervalForVariant(targetResults || null, item.variant) + if (!credibleInterval) { + return <>— + } + const [lowerBound, upperBound] = credibleInterval + return ( +
{`[${lowerBound > 0 ? '+' : ''}${lowerBound.toFixed( + 2 + )}%, ${upperBound > 0 ? '+' : ''}${upperBound.toFixed(2)}%]`}
+ ) + }, + }, { title: 'Win probability', render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element { diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx index 37e1eba587747..5ec2c1a5fe246 100644 --- a/frontend/src/scenes/experiments/experimentLogic.tsx +++ b/frontend/src/scenes/experiments/experimentLogic.tsx @@ -105,6 +105,18 @@ export interface ExperimentResultCalculationError { statusCode: number } +export interface CachedSecondaryMetricExperimentFunnelsQueryResponse extends CachedExperimentFunnelsQueryResponse { + filters?: { + insight?: InsightType + } +} + +export interface CachedSecondaryMetricExperimentTrendsQueryResponse extends CachedExperimentTrendsQueryResponse { + filters?: { + insight?: InsightType + } +} + export const experimentLogic = kea([ props({} as ExperimentLogicProps), key((props) => props.experimentId || 'new'), @@ -1261,6 +1273,53 @@ export const experimentLogic = kea([ return (variantResults[variantResults.length - 1].count / variantResults[0].count) * 100 }, ], + credibleIntervalForVariant: [ + () => [], + () => + ( + experimentResults: + | Partial + | CachedSecondaryMetricExperimentFunnelsQueryResponse + | CachedSecondaryMetricExperimentTrendsQueryResponse + | null, + variantKey: string + ): [number, number] | null => { + const credibleInterval = experimentResults?.credible_intervals?.[variantKey] + if (!credibleInterval) { + return null + } + + if (experimentResults.filters?.insight === InsightType.FUNNELS) { + const controlVariant = (experimentResults.variants as FunnelExperimentVariant[]).find( + ({ key }) => key === 'control' + ) as FunnelExperimentVariant + const controlConversionRate = + controlVariant.success_count / (controlVariant.success_count + controlVariant.failure_count) + + if (!controlConversionRate) { + return null + } + + // Calculate the percentage difference between the credible interval bounds of the variant and the control's conversion rate. + // This represents the range in which the true percentage change relative to the control is likely to fall. + const lowerBound = ((credibleInterval[0] - controlConversionRate) / controlConversionRate) * 100 + const upperBound = ((credibleInterval[1] - controlConversionRate) / controlConversionRate) * 100 + return [lowerBound, upperBound] + } + + const controlVariant = (experimentResults.variants as TrendExperimentVariant[]).find( + ({ key }) => key === 'control' + ) as TrendExperimentVariant + + const controlMean = controlVariant.count / controlVariant.absolute_exposure + + // Calculate the percentage difference between the credible interval bounds of the variant and the control's mean. + // This represents the range in which the true percentage change relative to the control is likely to fall. + const lowerBound = ((credibleInterval[0] - controlMean) / controlMean) * 100 + const upperBound = ((credibleInterval[1] - controlMean) / controlMean) * 100 + return [lowerBound, upperBound] + }, + ], getIndexForVariant: [ (s) => [s.experimentInsightType], (experimentInsightType) =>