feat(experiment): advice banners

frontend/src/scenes/experiments/ExperimentView/ProgressBar.tsx

@@ -1,32 +1,28 @@
 import '../Experiment.scss'
 
 import { useValues } from 'kea'
-import { dayjs } from 'lib/dayjs'
 import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
 import { humanFriendlyNumber } from 'lib/utils'
 
-import { FunnelStep, InsightType } from '~/types'
+import { InsightType } from '~/types'
 
 import { experimentLogic } from '../experimentLogic'
+import { formatUnitByQuantity } from '../utils'
 
 export function ProgressBar(): JSX.Element {
-    const { experiment, experimentResults, experimentInsightType } = useValues(experimentLogic)
-
-    // Parameters for experiment results
-    // don't use creation variables in results
-    const funnelResultsPersonsTotal =
-        experimentInsightType === InsightType.FUNNELS && experimentResults?.insight
-            ? (experimentResults.insight as FunnelStep[][]).reduce(
-                  (sum: number, variantResult: FunnelStep[]) => variantResult[0]?.count + sum,
-                  0
-              )
-            : 0
+    const {
+        experiment,
+        experimentInsightType,
+        funnelResultsPersonsTotal,
+        recommendedSampleSize,
+        actualRunningTime,
+        recommendedRunningTime,
+    } = useValues(experimentLogic)
 
     const experimentProgressPercent =
         experimentInsightType === InsightType.FUNNELS
-            ? ((funnelResultsPersonsTotal || 0) / (experiment?.parameters?.recommended_sample_size || 1)) * 100
-            : (dayjs().diff(experiment?.start_date, 'day') / (experiment?.parameters?.recommended_running_time || 1)) *
-              100
+            ? (funnelResultsPersonsTotal / recommendedSampleSize) * 100
+            : (actualRunningTime / recommendedRunningTime) * 100
 
     return (
         <div>
@@ -48,32 +44,34 @@ export function ProgressBar(): JSX.Element {
                 <div className="flex justify-between mt-2">
                     {experiment.end_date ? (
                         <div>
-                            Ran for <b>{dayjs(experiment.end_date).diff(experiment.start_date, 'day')}</b> days
+                            Ran for <b>{actualRunningTime}</b> {formatUnitByQuantity(actualRunningTime, 'day')}
                         </div>
                     ) : (
                         <div>
-                            <b>{dayjs().diff(experiment.start_date, 'day')}</b> days running
+                            <b>{actualRunningTime}</b> {formatUnitByQuantity(actualRunningTime, 'day')} running
                         </div>
                     )}
                     <div>
-                        Goal: <b>{experiment?.parameters?.recommended_running_time ?? 'Unknown'}</b> days
+                        Goal: <b>{recommendedRunningTime}</b> {formatUnitByQuantity(recommendedRunningTime, 'day')}
                     </div>
                 </div>
             )}
             {experimentInsightType === InsightType.FUNNELS && (
                 <div className="flex justify-between mt-2">
                     {experiment.end_date ? (
                         <div>
-                            Saw <b>{humanFriendlyNumber(funnelResultsPersonsTotal)}</b> participants
+                            Saw <b>{humanFriendlyNumber(funnelResultsPersonsTotal)}</b>{' '}
+                            {formatUnitByQuantity(funnelResultsPersonsTotal, 'participant')}
                         </div>
                     ) : (
                         <div>
-                            <b>{humanFriendlyNumber(funnelResultsPersonsTotal)}</b> participants seen
+                            <b>{humanFriendlyNumber(funnelResultsPersonsTotal)}</b>{' '}
+                            {formatUnitByQuantity(funnelResultsPersonsTotal, 'participant')} seen
                         </div>
                     )}
                     <div>
-                        Goal: <b>{humanFriendlyNumber(experiment?.parameters?.recommended_sample_size || 0)}</b>{' '}
-                        participants
+                        Goal: <b>{humanFriendlyNumber(recommendedSampleSize)}</b>{' '}
+                        {formatUnitByQuantity(recommendedSampleSize, 'participant')}
                     </div>
                 </div>
             )}

frontend/src/scenes/experiments/ExperimentView/components.tsx

@@ -15,7 +15,7 @@ import { urls } from 'scenes/urls'
 import { filtersToQueryNode } from '~/queries/nodes/InsightQuery/utils/filtersToQueryNode'
 import { Query } from '~/queries/Query/Query'
 import { NodeKind } from '~/queries/schema'
-import { ExperimentResults, InsightShortId } from '~/types'
+import { ExperimentResults, InsightShortId, InsightType } from '~/types'
 
 import { ResetButton } from '../Experiment'
 import { experimentLogic } from '../experimentLogic'
@@ -237,12 +237,18 @@ export function PageHeaderCustom(): JSX.Element {
 export function ActionBanner(): JSX.Element {
     const {
         experiment,
+        experimentInsightType,
         experimentResults,
         experimentLoading,
         experimentResultsLoading,
         isExperimentRunning,
         areResultsSignificant,
         isExperimentStopped,
+        funnelResultsPersonsTotal,
+        recommendedSampleSize,
+        actualRunningTime,
+        recommendedRunningTime,
+        getHighestProbabilityVariant,
     } = useValues(experimentLogic)
 
     if (!experiment || experimentLoading || experimentResultsLoading) {
@@ -263,18 +269,31 @@ export function ActionBanner(): JSX.Element {
         )
     }
 
-    // Trend: running, results present, not significant
-    if (isExperimentRunning && experimentResults && !isExperimentStopped && !areResultsSignificant) {
-        return (
-            <LemonBanner type="info" className="mt-4">
-                Your experiment is live and is collecting data, but hasn't yet reached the statistical significance
-                needed to make reliable decisions. It's important to wait for more data to avoid premature conclusions.
-            </LemonBanner>
-        )
-    }
-
     // Running, results present, not significant
     if (isExperimentRunning && experimentResults && !isExperimentStopped && !areResultsSignificant) {
+        // Results insignificant, but a large enough sample/running time has been achieved
+        // Further collection unlikely to change the result -> recommmend cutting the losses
+        if (
+            experimentInsightType === InsightType.FUNNELS &&
+            funnelResultsPersonsTotal > Math.min(recommendedSampleSize, 500)
+        ) {
+            return (
+                <LemonBanner type="warning" className="mt-4">
+                    You've reached a robust sample size for your experiment, but the results are still inconclusive.
+                    Continuing the experiment is unlikely to yield significant findings. It may be time to stop this
+                    experiment.
+                </LemonBanner>
+            )
+        }
+        if (experimentInsightType === InsightType.TRENDS && actualRunningTime > Math.min(recommendedRunningTime, 7)) {
+            return (
+                <LemonBanner type="warning" className="mt-4">
+                    Your experiment has been running long enough, but the results are still inconclusive. Continuing the
+                    experiment is unlikely to yield significant findings. It may be time to stop this experiment.
+                </LemonBanner>
+            )
+        }
+
         return (
             <LemonBanner type="info" className="mt-4">
                 Your experiment is live and is collecting data, but hasn't yet reached the statistical significance
@@ -284,7 +303,44 @@ export function ActionBanner(): JSX.Element {
     }
 
     // Running, results significant
-    if (isExperimentRunning && !isExperimentStopped && areResultsSignificant) {
+    if (isExperimentRunning && !isExperimentStopped && areResultsSignificant && experimentResults) {
+        const { probability } = experimentResults
+        const winningVariant = getHighestProbabilityVariant(experimentResults)
+        if (!winningVariant) {
+            return <></>
+        }
+
+        const winProbability = probability[winningVariant]
+
+        // Win probability only slightly over 0.9 and the recommended sample/time just met -> proceed with caution
+        if (
+            experimentInsightType === InsightType.FUNNELS &&
+            funnelResultsPersonsTotal > Math.min(recommendedSampleSize) + 50 &&
+            winProbability < 0.91
+        ) {
+            return (
+                <LemonBanner type="info" className="mt-4">
+                    You've achieved significant results, however, the sample size just meets the minimum requirements,
+                    and the win probability is only marginally above 90%. To ensure more reliable outcomes, consider
+                    running the experiment a bit longer.
+                </LemonBanner>
+            )
+        }
+
+        if (
+            experimentInsightType === InsightType.TRENDS &&
+            actualRunningTime > Math.min(recommendedRunningTime) + 2 &&
+            winProbability < 0.91
+        ) {
+            return (
+                <LemonBanner type="info" className="mt-4">
+                    You've achieved significant results, however, the running time just meets the minimum requirements,
+                    and the win probability is only marginally above 90%. To ensure more reliable outcomes, consider
+                    running the experiment a bit longer.
+                </LemonBanner>
+            )
+        }
+
         return (
             <LemonBanner type="success" className="mt-4">
                 Good news! Your experiment has gathered enough data to reach statistical significance, providing

frontend/src/scenes/experiments/experimentLogic.tsx

@@ -1102,6 +1102,44 @@ export const experimentLogic = kea<experimentLogicType>([
                 })
             },
         ],
+        recommendedSampleSize: [
+            (s) => [s.experiment],
+            (experiment: Experiment): number => experiment?.parameters?.recommended_sample_size || 100,
+        ],
+        funnelResultsPersonsTotal: [
+            (s) => [s.experimentResults, s.experimentInsightType],
+            (experimentResults: ExperimentResults['result'], experimentInsightType: InsightType): number => {
+                if (experimentInsightType !== InsightType.FUNNELS || !experimentResults?.insight) {
+                    return 0
+                }
+
+                let sum = 0
+                experimentResults.insight.forEach((variantResult) => {
+                    if (variantResult[0]?.count) {
+                        sum += variantResult[0].count
+                    }
+                })
+                return sum
+            },
+        ],
+        actualRunningTime: [
+            (s) => [s.experiment],
+            (experiment: Experiment): number => {
+                if (!experiment.start_date) {
+                    return 0
+                }
+
+                if (experiment.end_date) {
+                    return dayjs(experiment.end_date).diff(experiment.start_date, 'day')
+                }
+
+                return dayjs().diff(experiment.start_date, 'day')
+            },
+        ],
+        recommendedRunningTime: [
+            (s) => [s.experiment],
+            (experiment: Experiment): number => experiment?.parameters?.recommended_running_time || 1,
+        ],
     }),
     forms(({ actions, values }) => ({
         experiment: {

frontend/src/scenes/experiments/utils.ts

@@ -17,3 +17,7 @@ export const transformResultFilters = (filters: Partial<FilterType>): Partial<Fi
         display: ChartDisplayType.ActionsLineGraphCumulative,
     }),
 })
+
+export function formatUnitByQuantity(value: number, unit: string): string {
+    return value === 1 ? unit : unit + 's'
+}