feat(survey): Randomize questions & choices in surveys (#22227)

Users want to remove biases in the survey responses by shuffling around the questions when their customers get survey requests. 
To support this, we will add options to the API that allow us to save values specific to questions `shuffleOptions`, and a boolean flag `shuffleQuestions` to the survey object so that posthog-js can do the right thing based on these flags.

1. Adds a `shuffle options` checkbox to the UI that adds additional options to the questions. 
2. Adds a `Shuffle questions` checkbox to the `Customization` section of the survey edit UI.

frontend/src/scenes/surveys/SurveyCustomization.tsx

@@ -117,6 +117,18 @@ export function Customization({ appearance, surveyQuestionItem, onAppearanceChan
                         checked={appearance?.whiteLabel}
                     />
                 </div>
+
+                <div className="mt-2">
+                    <LemonCheckbox
+                        label={
+                            <div className="flex items-center">
+                                <span>Shuffle questions</span>
+                            </div>
+                        }
+                        onChange={(checked) => onAppearanceChange({ ...appearance, shuffleQuestions: checked })}
+                        checked={appearance?.shuffleQuestions}
+                    />
+                </div>
             </div>
         </>
     )

frontend/src/scenes/surveys/SurveyEditQuestionRow.tsx

@@ -278,6 +278,20 @@ export function SurveyEditQuestionGroup({ index, question }: { index: number; qu
                                                                 Add open-ended choice
                                                             </LemonButton>
                                                         )}
+                                                        <LemonField name="shuffleOptions" className="mt-2">
+                                                            {({
+                                                                value: shuffleOptions,
+                                                                onChange: toggleShuffleOptions,
+                                                            }) => (
+                                                                <LemonCheckbox
+                                                                    checked={!!shuffleOptions}
+                                                                    label="Shuffle options"
+                                                                    onChange={(checked) =>
+                                                                        toggleShuffleOptions(checked)
+                                                                    }
+                                                                />
+                                                            )}
+                                                        </LemonField>
                                                     </>
                                                 )}
                                             </div>

frontend/src/scenes/surveys/Surveys.stories.tsx

@@ -8,6 +8,7 @@ import { mswDecorator } from '~/mocks/browser'
 import { toPaginatedResponse } from '~/mocks/handlers'
 import {
     FeatureFlagBasicType,
+    MultipleSurveyQuestion,
     PropertyFilterType,
     PropertyOperator,
     Survey,
@@ -43,6 +44,44 @@ const MOCK_BASIC_SURVEY: Survey = {
     responses_limit: null,
 }
 
+const MOCK_SURVEY_WITH_MULTIPLE_OPTIONS: Survey = {
+    id: '998FE805-F9EF-4F25-A5D1-B9549C4E2143',
+    name: 'survey with multiple options',
+    description: 'survey with multiple options description',
+    type: SurveyType.Popover,
+    created_at: '2023-04-27T10:04:37.977401Z',
+    created_by: {
+        id: 1,
+        uuid: '01863799-062b-0000-8a61-b2842d5f8642',
+        distinct_id: 'Sopz9Z4NMIfXGlJe6W1XF98GOqhHNui5J5eRe0tBGTE',
+        first_name: 'Employee 427',
+        email: '[email protected]',
+    },
+    questions: [
+        {
+            type: SurveyQuestionType.MultipleChoice,
+            question: "We're sorry to see you go. What's your reason for unsubscribing?",
+            choices: [
+                'I no longer need the product',
+                'I found a better product',
+                'I found the product too difficult to use',
+                'Other',
+            ],
+            shuffleOptions: true,
+        },
+    ],
+    conditions: null,
+    linked_flag: null,
+    linked_flag_id: null,
+    targeting_flag: null,
+    targeting_flag_filters: undefined,
+    appearance: { backgroundColor: 'white', submitButtonColor: '#2C2C2C' },
+    start_date: null,
+    end_date: null,
+    archived: false,
+    responses_limit: null,
+}
+
 const MOCK_SURVEY_WITH_RELEASE_CONS: Survey = {
     id: '0187c279-bcae-0000-34f5-4f121921f006',
     name: 'survey with release conditions',
@@ -154,9 +193,12 @@ const meta: Meta = {
                 '/api/projects/:team_id/surveys/': toPaginatedResponse([
                     MOCK_BASIC_SURVEY,
                     MOCK_SURVEY_WITH_RELEASE_CONS,
+                    MOCK_SURVEY_WITH_MULTIPLE_OPTIONS,
                 ]),
                 '/api/projects/:team_id/surveys/0187c279-bcae-0000-34f5-4f121921f005/': MOCK_BASIC_SURVEY,
                 '/api/projects/:team_id/surveys/0187c279-bcae-0000-34f5-4f121921f006/': MOCK_SURVEY_WITH_RELEASE_CONS,
+                '/api/projects/:team_id/surveys/998FE805-F9EF-4F25-A5D1-B9549C4E2143/':
+                    MOCK_SURVEY_WITH_MULTIPLE_OPTIONS,
                 '/api/projects/:team_id/surveys/responses_count/': MOCK_RESPONSES_COUNT,
                 [`/api/projects/:team_id/feature_flags/${
                     (MOCK_SURVEY_WITH_RELEASE_CONS.linked_flag as FeatureFlagBasicType).id
@@ -206,6 +248,27 @@ export const NewSurveyCustomisationSection: StoryFn = () => {
     return <App />
 }
 
+export const NewMultiQuestionSurveySection: StoryFn = () => {
+    useEffect(() => {
+        router.actions.push(urls.survey('new'))
+        surveyLogic({ id: 'new' }).mount()
+        surveyLogic({ id: 'new' }).actions.setSelectedSection(SurveyEditSection.Steps)
+        surveyLogic({ id: 'new' }).actions.setSurveyValue('questions', [
+            {
+                type: SurveyQuestionType.MultipleChoice,
+                question: "We're sorry to see you go. What's your reason for unsubscribing?",
+                choices: [
+                    'I no longer need the product',
+                    'I found a better product',
+                    'I found the product too difficult to use',
+                    'Other',
+                ],
+            } as MultipleSurveyQuestion,
+        ])
+    }, [])
+    return <App />
+}
+
 export const NewSurveyPresentationSection: StoryFn = () => {
     useEffect(() => {
         router.actions.push(urls.survey('new'))

frontend/src/types.ts

@@ -2497,6 +2497,7 @@ export interface SurveyAppearance {
     widgetSelector?: string
     widgetLabel?: string
     widgetColor?: string
+    shuffleQuestions?: boolean
 }
 
 export interface SurveyQuestionBase {
@@ -2526,6 +2527,7 @@ export interface RatingSurveyQuestion extends SurveyQuestionBase {
 export interface MultipleSurveyQuestion extends SurveyQuestionBase {
     type: SurveyQuestionType.SingleChoice | SurveyQuestionType.MultipleChoice
     choices: string[]
+    shuffleOptions?: boolean
     hasOpenChoice?: boolean
 }
 

posthog/api/test/test_survey.py

@@ -1009,6 +1009,70 @@ def test_surveys_opt_in_post_delete(self):
         assert self.team.surveys_opt_in is False
 
 
+class TestMultipleChoiceQuestions(APIBaseTest):
+    def test_create_survey_has_open_choice(self):
+        response = self.client.post(
+            f"/api/projects/{self.team.id}/surveys/",
+            data={
+                "name": "Notebooks beta release survey",
+                "description": "Get feedback on the new notebooks feature",
+                "type": "popover",
+                "questions": [
+                    {
+                        "type": "multiple_choice",
+                        "choices": ["Tutorials", "Customer case studies", "Product announcements", "Other"],
+                        "question": "What can we do to improve our product?",
+                        "buttonText": "Submit",
+                        "description": "",
+                        "hasOpenChoice": True,
+                    }
+                ],
+                "appearance": {
+                    "thankYouMessageHeader": "Thanks for your feedback!",
+                    "thankYouMessageDescription": "<b>We'll use it to make notebooks better.<script>alert(0)</script>",
+                },
+            },
+            format="json",
+        )
+        response_data = response.json()
+        assert response.status_code == status.HTTP_201_CREATED, response_data
+        assert Survey.objects.filter(id=response_data["id"]).exists()
+        assert response_data["name"] == "Notebooks beta release survey"
+        assert response_data["questions"][0]["hasOpenChoice"] is True
+
+    def test_create_survey_with_shuffle_options(self):
+        response = self.client.post(
+            f"/api/projects/{self.team.id}/surveys/",
+            data={
+                "name": "Notebooks beta release survey",
+                "description": "Get feedback on the new notebooks feature",
+                "type": "popover",
+                "questions": [
+                    {
+                        "type": "multiple_choice",
+                        "choices": ["Tutorials", "Customer case studies", "Product announcements", "Other"],
+                        "question": "What can we do to improve our product?",
+                        "buttonText": "Submit",
+                        "description": "",
+                        "hasOpenChoice": True,
+                        "shuffleOptions": True,
+                    }
+                ],
+                "appearance": {
+                    "thankYouMessageHeader": "Thanks for your feedback!",
+                    "thankYouMessageDescription": "<b>We'll use it to make notebooks better.<script>alert(0)</script>",
+                },
+            },
+            format="json",
+        )
+        response_data = response.json()
+        assert response.status_code == status.HTTP_201_CREATED, response_data
+        assert Survey.objects.filter(id=response_data["id"]).exists()
+        assert response_data["name"] == "Notebooks beta release survey"
+        assert response_data["questions"][0]["hasOpenChoice"] is True
+        assert response_data["questions"][0]["shuffleOptions"] is True
+
+
 class TestSurveyQuestionValidation(APIBaseTest):
     def test_create_basic_survey_question_validation(self):
         response = self.client.post(
@@ -1032,6 +1096,7 @@ def test_create_basic_survey_question_validation(self):
                 "appearance": {
                     "thankYouMessageHeader": "Thanks for your feedback!",
                     "thankYouMessageDescription": "<b>We'll use it to make notebooks better.<script>alert(0)</script>",
+                    "shuffleQuestions": True,
                 },
             },
             format="json",
@@ -1053,6 +1118,7 @@ def test_create_basic_survey_question_validation(self):
         assert response_data["appearance"] == {
             "thankYouMessageHeader": "Thanks for your feedback!",
             "thankYouMessageDescription": "<b>We'll use it to make notebooks better.</b>",
+            "shuffleQuestions": True,
         }
         assert response_data["created_by"]["id"] == self.user.id