feat(experiment): add details for "Not significant" (#21596)

PostHog · Apr 17, 2024 · 1cc3d5a · 1cc3d5a
1 parent 880fb70
commit 1cc3d5a
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 2 deletions.
diff --git a/frontend/src/scenes/experiments/ExperimentView/components.tsx b/frontend/src/scenes/experiments/ExperimentView/components.tsx
@@ -1,7 +1,7 @@
 import '../Experiment.scss'
 
 import { IconCheck, IconX } from '@posthog/icons'
-import { LemonBanner, LemonButton, LemonDivider, LemonTag, LemonTagType, Link } from '@posthog/lemon-ui'
+import { LemonBanner, LemonButton, LemonDivider, LemonTag, LemonTagType, Link, Tooltip } from '@posthog/lemon-ui'
 import { Empty } from 'antd'
 import { useActions, useValues } from 'kea'
 import { AnimationType } from 'lib/animations/animations'
@@ -41,11 +41,21 @@ export function VariantTag({ variantKey }: { variantKey: string }): JSX.Element
 }
 
 export function ResultsTag(): JSX.Element {
-    const { areResultsSignificant } = useValues(experimentLogic)
+    const { areResultsSignificant, significanceDetails } = useValues(experimentLogic)
     const result: { color: LemonTagType; label: string } = areResultsSignificant
         ? { color: 'success', label: 'Significant' }
         : { color: 'primary', label: 'Not significant' }
 
+    if (significanceDetails) {
+        return (
+            <Tooltip title={significanceDetails}>
+                <LemonTag className="cursor-pointer" type={result.color}>
+                    <b className="uppercase">{result.label}</b>
+                </LemonTag>
+            </Tooltip>
+        )
+    }
+
     return (
         <LemonTag type={result.color}>
             <b className="uppercase">{result.label}</b>

diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx
@@ -773,6 +773,7 @@ export const experimentLogic = kea<experimentLogicType>([
                 return experimentResults?.significant || false
             },
         ],
+        // TODO: remove with the old UI
         significanceBannerDetails: [
             (s) => [s.experimentResults],
             (experimentResults): string | ReactElement => {
@@ -819,6 +820,32 @@ export const experimentLogic = kea<experimentLogicType>([
                 return ''
             },
         ],
+        significanceDetails: [
+            (s) => [s.experimentResults],
+            (experimentResults): string => {
+                if (experimentResults?.significance_code === SignificanceCode.HighLoss) {
+                    return `This is because the expected loss in conversion is greater than 1% (current value is ${(
+                        (experimentResults?.expected_loss || 0) * 100
+                    )?.toFixed(2)}%).`
+                }
+
+                if (experimentResults?.significance_code === SignificanceCode.HighPValue) {
+                    return `This is because the p value is greater than 0.05 (current value is ${
+                        experimentResults?.p_value?.toFixed(3) || 1
+                    }).`
+                }
+
+                if (experimentResults?.significance_code === SignificanceCode.LowWinProbability) {
+                    return 'This is because the win probability of all test variants combined is less than 90%.'
+                }
+
+                if (experimentResults?.significance_code === SignificanceCode.NotEnoughExposure) {
+                    return 'This is because we need at least 100 people per variant to declare significance.'
+                }
+
+                return ''
+            },
+        ],
         recommendedExposureForCountData: [
             (s) => [s.minimumDetectableChange],
             (mde) =>