feat(experiments): Allow up to 19 variants for an experiment (#27223)

Co-authored-by: Neil Kakkar <[email protected]>

ee/clickhouse/views/experiments.py

@@ -243,8 +243,8 @@ def validate_parameters(self, value):
 
         variants = value.get("feature_flag_variants", [])
 
-        if len(variants) >= 11:
-            raise ValidationError("Feature flag variants must be less than 11")
+        if len(variants) >= 21:
+            raise ValidationError("Feature flag variants must be less than 21")
         elif len(variants) > 0:
             if "control" not in [variant["key"] for variant in variants]:
                 raise ValidationError("Feature flag variants must contain a control variant")

frontend/src/lib/constants.tsx

@@ -107,7 +107,7 @@ export const INSTANTLY_AVAILABLE_PROPERTIES = [
     '$group_key',
     'distinct_id',
 ]
-export const MAX_EXPERIMENT_VARIANTS = 10
+export const MAX_EXPERIMENT_VARIANTS = 20
 export const EXPERIMENT_DEFAULT_DURATION = 14 // days
 
 // Event constants

frontend/src/scenes/experiments/ExperimentForm.tsx

@@ -139,7 +139,9 @@ const ExperimentFormFields = (): JSX.Element => {
                 )}
                 <div className="mt-10">
                     <h3 className="mb-1">Variants</h3>
-                    <div className="text-xs text-muted">Add up to 9 variants to test against your control.</div>
+                    <div className="text-xs text-muted">
+                        Add up to {MAX_EXPERIMENT_VARIANTS - 1} variants to test against your control.
+                    </div>
                     <LemonDivider />
                     <div className="grid grid-cols-2 gap-4 max-w-160">
                         <div className="max-w-60">

frontend/src/scenes/experiments/experimentLogic.test.ts

@@ -4,7 +4,7 @@ import { userLogic } from 'scenes/userLogic'
 import { useMocks } from '~/mocks/jest'
 import { initKeaTests } from '~/test/init'
 
-import { experimentLogic } from './experimentLogic'
+import { experimentLogic, percentageDistribution } from './experimentLogic'
 
 const RUNNING_EXP_ID = 45
 const RUNNING_FUNNEL_EXP_ID = 46
@@ -121,4 +121,29 @@ describe('experimentLogic', () => {
             expect(logic.values.recommendedExposureForCountData(0)).toEqual(Infinity)
         })
     })
+
+    describe('percentageDistribution', () => {
+        it('given variant count, calculates correct rollout percentages', async () => {
+            expect(percentageDistribution(1)).toEqual([100])
+            expect(percentageDistribution(2)).toEqual([50, 50])
+            expect(percentageDistribution(3)).toEqual([34, 33, 33])
+            expect(percentageDistribution(4)).toEqual([25, 25, 25, 25])
+            expect(percentageDistribution(5)).toEqual([20, 20, 20, 20, 20])
+            expect(percentageDistribution(6)).toEqual([17, 17, 17, 17, 16, 16])
+            expect(percentageDistribution(7)).toEqual([15, 15, 14, 14, 14, 14, 14])
+            expect(percentageDistribution(8)).toEqual([13, 13, 13, 13, 12, 12, 12, 12])
+            expect(percentageDistribution(9)).toEqual([12, 11, 11, 11, 11, 11, 11, 11, 11])
+            expect(percentageDistribution(10)).toEqual([10, 10, 10, 10, 10, 10, 10, 10, 10, 10])
+            expect(percentageDistribution(11)).toEqual([10, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9])
+            expect(percentageDistribution(12)).toEqual([9, 9, 9, 9, 8, 8, 8, 8, 8, 8, 8, 8])
+            expect(percentageDistribution(13)).toEqual([8, 8, 8, 8, 8, 8, 8, 8, 8, 7, 7, 7, 7])
+            expect(percentageDistribution(14)).toEqual([8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7])
+            expect(percentageDistribution(15)).toEqual([7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6])
+            expect(percentageDistribution(16)).toEqual([7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6])
+            expect(percentageDistribution(17)).toEqual([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5])
+            expect(percentageDistribution(18)).toEqual([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5])
+            expect(percentageDistribution(19)).toEqual([6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5])
+            expect(percentageDistribution(20)).toEqual([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5])
+        })
+    })
 })

frontend/src/scenes/experiments/experimentLogic.tsx

@@ -1787,12 +1787,14 @@ export const experimentLogic = kea<experimentLogicType>([
     })),
 ])
 
-function percentageDistribution(variantCount: number): number[] {
-    const percentageRounded = Math.round(100 / variantCount)
-    const totalRounded = percentageRounded * variantCount
-    const delta = totalRounded - 100
-    const percentages = new Array(variantCount).fill(percentageRounded)
-    percentages[variantCount - 1] = percentageRounded - delta
+export function percentageDistribution(variantCount: number): number[] {
+    const basePercentage = Math.floor(100 / variantCount)
+    const percentages = new Array(variantCount).fill(basePercentage)
+    let remaining = 100 - basePercentage * variantCount
+    for (let i = 0; remaining > 0; i++, remaining--) {
+        // try to equally distribute `remaining` across variants
+        percentages[i] += 1
+    }
     return percentages
 }