@@ -48,15 +44,15 @@ export function ProgressBar(): JSX.Element {
{experiment.end_date ? (
- Ran for {dayjs(experiment.end_date).diff(experiment.start_date, 'day')} days
+ Ran for {actualRunningTime} {formatUnitByQuantity(actualRunningTime, 'day')}
) : (
- {dayjs().diff(experiment.start_date, 'day')} days running
+ {actualRunningTime} {formatUnitByQuantity(actualRunningTime, 'day')} running
)}
diff --git a/frontend/src/scenes/experiments/ExperimentView/components.tsx b/frontend/src/scenes/experiments/ExperimentView/components.tsx
index 6f2d5dcecb004..6818e1d516256 100644
--- a/frontend/src/scenes/experiments/ExperimentView/components.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/components.tsx
@@ -1,7 +1,7 @@
import '../Experiment.scss'
import { IconCheckbox } from '@posthog/icons'
-import { LemonButton, LemonDivider, LemonTag, LemonTagType } from '@posthog/lemon-ui'
+import { LemonBanner, LemonButton, LemonDivider, LemonTag, LemonTagType, Link } from '@posthog/lemon-ui'
import { Empty } from 'antd'
import { useActions, useValues } from 'kea'
import { AnimationType } from 'lib/animations/animations'
@@ -335,3 +335,159 @@ export function PageHeaderCustom(): JSX.Element {
/>
)
}
+
+export function ActionBanner(): JSX.Element {
+ const {
+ experiment,
+ experimentInsightType,
+ experimentResults,
+ experimentLoading,
+ experimentResultsLoading,
+ isExperimentRunning,
+ areResultsSignificant,
+ isExperimentStopped,
+ funnelResultsPersonsTotal,
+ recommendedSampleSize,
+ actualRunningTime,
+ recommendedRunningTime,
+ getHighestProbabilityVariant,
+ } = useValues(experimentLogic)
+
+ const { archiveExperiment } = useActions(experimentLogic)
+
+ if (!experiment || experimentLoading || experimentResultsLoading) {
+ return <>
+ }
+
+ // Draft
+ if (!isExperimentRunning) {
+ return (
+
+ Your experiment is in draft mode. You can edit your variants, adjust release conditions, and{' '}
+
+ test your feature flag
+
+ . Once everything works as expected, you can launch your experiment. From that point, any new experiment
+ events will be counted towards the results.
+
+ )
+ }
+
+ // Running, results present, not significant
+ if (isExperimentRunning && experimentResults && !isExperimentStopped && !areResultsSignificant) {
+ // Results insignificant, but a large enough sample/running time has been achieved
+ // Further collection unlikely to change the result -> recommmend cutting the losses
+ if (
+ experimentInsightType === InsightType.FUNNELS &&
+ funnelResultsPersonsTotal > Math.max(recommendedSampleSize, 500) &&
+ dayjs().diff(experiment.start_date, 'day') > 2 // at least 2 days running
+ ) {
+ return (
+
+ You've reached a robust sample size for your experiment, but the results are still inconclusive.
+ Continuing the experiment is unlikely to yield significant findings. It may be time to stop this
+ experiment.
+
+ )
+ }
+ if (experimentInsightType === InsightType.TRENDS && actualRunningTime > Math.max(recommendedRunningTime, 7)) {
+ return (
+
+ Your experiment has been running long enough, but the results are still inconclusive. Continuing the
+ experiment is unlikely to yield significant findings. It may be time to stop this experiment.
+
+ )
+ }
+
+ return (
+
+ Your experiment is live and is collecting data, but hasn't yet reached the statistical significance
+ needed to make reliable decisions. It's important to wait for more data to avoid premature conclusions.
+
+ )
+ }
+
+ // Running, results significant
+ if (isExperimentRunning && !isExperimentStopped && areResultsSignificant && experimentResults) {
+ const { probability } = experimentResults
+ const winningVariant = getHighestProbabilityVariant(experimentResults)
+ if (!winningVariant) {
+ return <>
+ }
+
+ const winProbability = probability[winningVariant]
+
+ // Win probability only slightly over 0.9 and the recommended sample/time just met -> proceed with caution
+ if (
+ experimentInsightType === InsightType.FUNNELS &&
+ funnelResultsPersonsTotal > recommendedSampleSize + 50 &&
+ winProbability < 0.93
+ ) {
+ return (
+
+ You've achieved significant results, however, the sample size just meets the minimum requirements,
+ and the win probability is only marginally above 90%. To ensure more reliable outcomes, consider
+ running the experiment a bit longer.
+
+ )
+ }
+
+ if (
+ experimentInsightType === InsightType.TRENDS &&
+ actualRunningTime > recommendedRunningTime + 2 &&
+ winProbability < 0.93
+ ) {
+ return (
+
+ You've achieved significant results, however, the running time just meets the minimum requirements,
+ and the win probability is only marginally above 90%. To ensure more reliable outcomes, consider
+ running the experiment a bit longer.
+
+ )
+ }
+
+ return (
+
+ Good news! Your experiment has gathered enough data to reach statistical significance, providing
+ reliable results for decision making. Before taking any action, review relevant secondary metrics for
+ any unintended side effects. Once you're done, you can stop the experiment.
+
+ )
+ }
+
+ // Stopped, results significant
+ if (isExperimentStopped && areResultsSignificant) {
+ return (
+
+ You have stopped this experiment, and it is no longer collecting data. With significant results in hand,
+ you can now roll out the winning variant to all your users by adjusting the{' '}
+
+ {experiment.feature_flag?.key}
+ {' '}
+ feature flag.
+
+ )
+ }
+
+ // Stopped, results not significant
+ if (isExperimentStopped && experimentResults && !areResultsSignificant) {
+ return (
+
+ You have stopped this experiment, and it is no longer collecting data. Because your results are not
+ significant, we don't recommend drawing any conclusions from them. You can reset the experiment
+ (deleting the data collected so far) and restart the experiment at any point again. If this experiment
+ is no longer relevant, you can{' '}
+ archiveExperiment()}>
+ archive it
+
+ .
+
+ )
+ }
+
+ return <>
+}
diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx
index 7b2b10ed36540..31de23b9d696f 100644
--- a/frontend/src/scenes/experiments/experimentLogic.tsx
+++ b/frontend/src/scenes/experiments/experimentLogic.tsx
@@ -1107,6 +1107,44 @@ export const experimentLogic = kea([
})
},
],
+ recommendedSampleSize: [
+ (s) => [s.experiment],
+ (experiment: Experiment): number => experiment?.parameters?.recommended_sample_size || 100,
+ ],
+ funnelResultsPersonsTotal: [
+ (s) => [s.experimentResults, s.experimentInsightType],
+ (experimentResults: ExperimentResults['result'], experimentInsightType: InsightType): number => {
+ if (experimentInsightType !== InsightType.FUNNELS || !experimentResults?.insight) {
+ return 0
+ }
+
+ let sum = 0
+ experimentResults.insight.forEach((variantResult) => {
+ if (variantResult[0]?.count) {
+ sum += variantResult[0].count
+ }
+ })
+ return sum
+ },
+ ],
+ actualRunningTime: [
+ (s) => [s.experiment],
+ (experiment: Experiment): number => {
+ if (!experiment.start_date) {
+ return 0
+ }
+
+ if (experiment.end_date) {
+ return dayjs(experiment.end_date).diff(experiment.start_date, 'day')
+ }
+
+ return dayjs().diff(experiment.start_date, 'day')
+ },
+ ],
+ recommendedRunningTime: [
+ (s) => [s.experiment],
+ (experiment: Experiment): number => experiment?.parameters?.recommended_running_time || 1,
+ ],
}),
forms(({ actions, values }) => ({
experiment: {
diff --git a/frontend/src/scenes/experiments/utils.ts b/frontend/src/scenes/experiments/utils.ts
index 90d7b2c64f44b..6f71d6c1829b2 100644
--- a/frontend/src/scenes/experiments/utils.ts
+++ b/frontend/src/scenes/experiments/utils.ts
@@ -17,3 +17,7 @@ export const transformResultFilters = (filters: Partial): Partial