Merge branch 'master' into fix/enforce-security-functions

# Conflicts: # posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr
PostHog · Apr 25, 2024 · c1a8e72 · c1a8e72
2 parents 3c182c0 + 1778528
commit c1a8e72
Show file tree

Hide file tree

Showing 47 changed files with 545 additions and 39 deletions.
diff --git a/docker-compose.hobby.yml b/docker-compose.hobby.yml
@@ -142,7 +142,7 @@ services:
     elasticsearch:
         extends:
             file: docker-compose.base.yml
-            service: elasticsearch            
+            service: elasticsearch
     temporal-admin-tools:
         extends:
             file: docker-compose.base.yml

diff --git a/frontend/__snapshots__/scenes-other-settings--settings-project--dark.png b/frontend/__snapshots__/scenes-other-settings--settings-project--dark.png
diff --git a/frontend/__snapshots__/scenes-other-settings--settings-project--light.png b/frontend/__snapshots__/scenes-other-settings--settings-project--light.png
diff --git a/frontend/src/lib/api.mock.ts b/frontend/src/lib/api.mock.ts
@@ -76,6 +76,7 @@ export const MOCK_DEFAULT_TEAM: TeamType = {
     session_replay_config: null,
     capture_console_log_opt_in: true,
     capture_performance_opt_in: true,
+    heatmaps_opt_in: true,
     autocapture_exceptions_opt_in: false,
     autocapture_exceptions_errors_to_ignore: [],
     effective_membership_level: OrganizationMembershipLevel.Admin,

diff --git a/frontend/src/lib/utils/eventUsageLogic.ts b/frontend/src/lib/utils/eventUsageLogic.ts
@@ -457,6 +457,7 @@ export const eventUsageLogic = kea<eventUsageLogicType>([
         reportIngestionContinueWithoutVerifying: true,
         reportAutocaptureToggled: (autocapture_opt_out: boolean) => ({ autocapture_opt_out }),
         reportAutocaptureExceptionsToggled: (autocapture_opt_in: boolean) => ({ autocapture_opt_in }),
+        reportHeatmapsToggled: (heatmaps_opt_in: boolean) => ({ heatmaps_opt_in }),
         reportFailedToCreateFeatureFlagWithCohort: (code: string, detail: string) => ({ code, detail }),
         reportFeatureFlagCopySuccess: true,
         reportFeatureFlagCopyFailure: (error) => ({ error }),
@@ -1103,6 +1104,11 @@ export const eventUsageLogic = kea<eventUsageLogicType>([
                 autocapture_opt_in,
             })
         },
+        reportHeatmapsToggled: ({ heatmaps_opt_in }) => {
+            posthog.capture('heatmaps toggled', {
+                heatmaps_opt_in,
+            })
+        },
         reportFailedToCreateFeatureFlagWithCohort: ({ detail, code }) => {
             posthog.capture('failed to create feature flag with cohort', { detail, code })
         },

diff --git a/frontend/src/lib/utils/getAppContext.ts b/frontend/src/lib/utils/getAppContext.ts
@@ -16,6 +16,11 @@ export function getDefaultEventName(): string {
     return getAppContext()?.default_event_name || PathType.PageView
 }
 
+export function getDefaultEventLabel(): string {
+    const name = getDefaultEventName()
+    return name === PathType.PageView ? 'Pageview' : name === PathType.Screen ? 'Screen' : name
+}
+
 // NOTE: Any changes to the teamId trigger a full page load so we don't use the logic
 // This helps avoid circular imports
 export function getCurrentTeamId(): TeamType['id'] {

diff --git a/frontend/src/scenes/data-warehouse/external/dataWarehouseSceneLogic.ts b/frontend/src/scenes/data-warehouse/external/dataWarehouseSceneLogic.ts
@@ -176,9 +176,9 @@ export const dataWarehouseSceneLogic = kea<dataWarehouseSceneLogicType>([
         selectRow: () => {
             actions.setIsEditingSavedQuery(false)
         },
-        updateDataWarehouseSavedQuerySuccess: async (_, view) => {
+        updateDataWarehouseSavedQuerySuccess: async ({ payload }) => {
             actions.setIsEditingSavedQuery(false)
-            lemonToast.success(`${view.name} successfully updated`)
+            lemonToast.success(`${payload?.name ?? 'View'} successfully updated`)
         },
     })),
     afterMount(({ actions, values }) => {

diff --git a/frontend/src/scenes/data-warehouse/external/forms/SyncProgressStep.tsx b/frontend/src/scenes/data-warehouse/external/forms/SyncProgressStep.tsx
@@ -3,28 +3,31 @@ import { useValues } from 'kea'
 import { sourceWizardLogic } from 'scenes/data-warehouse/new/sourceWizardLogic'
 import { dataWarehouseSettingsLogic } from 'scenes/data-warehouse/settings/dataWarehouseSettingsLogic'
 
+import { ExternalDataSourceSchema } from '~/types'
+
 export const SyncProgressStep = (): JSX.Element => {
-    const { databaseSchema, sourceId } = useValues(sourceWizardLogic)
+    const { sourceId } = useValues(sourceWizardLogic)
     const { dataWarehouseSources, dataWarehouseSourcesLoading } = useValues(dataWarehouseSettingsLogic)
 
     const source = dataWarehouseSources?.results.find((n) => n.id === sourceId)
+    const schemas = source?.schemas ?? []
 
-    const getSyncStatus = (shouldSync: boolean): { status: string; tagType: LemonTagType } => {
-        if (!shouldSync) {
+    const getSyncStatus = (schema: ExternalDataSourceSchema): { status: string; tagType: LemonTagType } => {
+        if (!schema.should_sync) {
             return {
                 status: 'Not synced',
                 tagType: 'default',
             }
         }
 
-        if (!source || source.status === 'Running') {
+        if (schema.status === 'Running') {
             return {
                 status: 'Syncing...',
                 tagType: 'primary',
             }
         }
 
-        if (source.status === 'Completed') {
+        if (schema.status === 'Completed') {
             return {
                 status: 'Completed',
                 tagType: 'success',
@@ -42,22 +45,22 @@ export const SyncProgressStep = (): JSX.Element => {
             <div>
                 <LemonTable
                     emptyState="No schemas selected"
-                    dataSource={databaseSchema}
+                    dataSource={schemas}
                     loading={dataWarehouseSourcesLoading}
                     disableTableWhileLoading={false}
                     columns={[
                         {
                             title: 'Table',
                             key: 'table',
                             render: function RenderTable(_, schema) {
-                                return schema.table
+                                return schema.name
                             },
                         },
                         {
                             title: 'Status',
                             key: 'status',
                             render: function RenderStatus(_, schema) {
-                                const { status, tagType } = getSyncStatus(schema.should_sync)
+                                const { status, tagType } = getSyncStatus(schema)
 
                                 return <LemonTag type={tagType}>{status}</LemonTag>
                             },

diff --git a/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx b/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx
@@ -448,6 +448,7 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
                 lemonToast.success('New Data Resource Created')
                 actions.setSourceId(id)
                 actions.resetSourceConnectionDetails()
+                actions.loadSources(null)
                 actions.onNext()
             } catch (e: any) {
                 lemonToast.error(e.data?.message ?? e.message)

diff --git a/frontend/src/scenes/insights/filters/ActionFilter/entityFilterLogic.ts b/frontend/src/scenes/insights/filters/ActionFilter/entityFilterLogic.ts
@@ -2,7 +2,7 @@ import { actions, connect, events, kea, key, listeners, path, props, reducers, s
 import { convertPropertyGroupToProperties } from 'lib/components/PropertyFilters/utils'
 import { uuid } from 'lib/utils'
 import { eventUsageLogic, GraphSeriesAddedSource } from 'lib/utils/eventUsageLogic'
-import { getDefaultEventName } from 'lib/utils/getAppContext'
+import { getDefaultEventLabel, getDefaultEventName } from 'lib/utils/getAppContext'
 import { insightDataLogic } from 'scenes/insights/insightDataLogic'
 
 import {
@@ -262,8 +262,9 @@ export const entityFilterLogic = kea<entityFilterLogicType>([
             const newLength = previousLength + 1
             const precedingEntity = values.localFilters[previousLength - 1] as LocalFilter | undefined
             const order = precedingEntity ? precedingEntity.order + 1 : 0
-            const newFilter = {
+            const newFilter: LocalFilter = {
                 id: getDefaultEventName(),
+                name: getDefaultEventLabel(),
                 uuid: uuid(),
                 type: EntityTypes.EVENTS,
                 order: order,

diff --git a/frontend/src/scenes/onboarding/Onboarding.tsx b/frontend/src/scenes/onboarding/Onboarding.tsx
@@ -1,5 +1,6 @@
 import { useActions, useValues } from 'kea'
 import { FEATURE_FLAGS, SESSION_REPLAY_MINIMUM_DURATION_OPTIONS } from 'lib/constants'
+import { useFeatureFlag } from 'lib/hooks/useFeatureFlag'
 import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
 import { useEffect, useState } from 'react'
 import { AndroidInstructions } from 'scenes/onboarding/sdks/session-replay'
@@ -81,6 +82,8 @@ const OnboardingWrapper = ({ children }: { children: React.ReactNode }): JSX.Ele
 const ProductAnalyticsOnboarding = (): JSX.Element => {
     const { currentTeam } = useValues(teamLogic)
 
+    const heatmapsEnabled = useFeatureFlag('TOOLBAR_HEATMAPS')
+
     return (
         <OnboardingWrapper>
             <SDKs
@@ -101,6 +104,18 @@ const ProductAnalyticsOnboarding = (): JSX.Element => {
                         type: 'toggle',
                         inverseToggle: true,
                     },
+
+                    heatmapsEnabled
+                        ? {
+                              title: 'Enable heatmaps',
+                              description: `If you use our JavaScript libraries, we can capture general clicks, mouse movements,
+                               and scrolling to create heatmaps. 
+                               No additional events are created, and you can disable this at any time.`,
+                              teamProperty: 'heatmaps_opt_in',
+                              value: currentTeam?.heatmaps_opt_in ?? true,
+                              type: 'toggle',
+                          }
+                        : undefined,
                 ]}
             />
         </OnboardingWrapper>

diff --git a/frontend/src/scenes/onboarding/OnboardingProductConfiguration.tsx b/frontend/src/scenes/onboarding/OnboardingProductConfiguration.tsx
@@ -44,7 +44,7 @@ export const OnboardingProductConfiguration = ({
     options,
 }: {
     stepKey?: OnboardingStepKey
-    options: ProductConfigOption[]
+    options: (ProductConfigOption | undefined)[]
 }): JSX.Element | null => {
     const { configOptions } = useValues(onboardingProductConfigurationLogic)
     const { defaultEnabledPlugins } = useValues(pluginsLogic)
@@ -58,7 +58,7 @@ export const OnboardingProductConfiguration = ({
     }, [configOptions])
 
     useEffect(() => {
-        setConfigOptions(options)
+        setConfigOptions(options.filter((option): option is ProductConfigOption => !!option))
     }, [])
 
     const combinedList: ConfigOption[] = [

diff --git a/frontend/src/scenes/onboarding/onboardingProductConfigurationLogic.ts b/frontend/src/scenes/onboarding/onboardingProductConfigurationLogic.ts
@@ -2,12 +2,14 @@ import { LemonSelectOptions } from '@posthog/lemon-ui'
 import { actions, connect, kea, listeners, path, reducers } from 'kea'
 import { teamLogic } from 'scenes/teamLogic'
 
+import { TeamType } from '~/types'
+
 import type { onboardingProductConfigurationLogicType } from './onboardingProductConfigurationLogicType'
 
 export interface ProductConfigOptionBase {
     title: string
     description: string
-    teamProperty: string
+    teamProperty: keyof TeamType
 }
 
 export interface ProductConfigurationToggle extends ProductConfigOptionBase {

diff --git a/frontend/src/scenes/settings/SettingsMap.tsx b/frontend/src/scenes/settings/SettingsMap.tsx
@@ -11,6 +11,7 @@ import { AutocaptureSettings, ExceptionAutocaptureSettings } from './project/Aut
 import { CorrelationConfig } from './project/CorrelationConfig'
 import { DataAttributes } from './project/DataAttributes'
 import { GroupAnalyticsConfig } from './project/GroupAnalyticsConfig'
+import { HeatmapsSettings } from './project/HeatmapsSettings'
 import { IPAllowListInfo } from './project/IPAllowListInfo'
 import { IPCapture } from './project/IPCapture'
 import { PathCleaningFiltersConfig } from './project/PathCleaningFiltersConfig'
@@ -76,14 +77,20 @@ export const SettingsMap: SettingSection[] = [
     {
         level: 'project',
         id: 'project-autocapture',
-        title: 'Autocapture',
+        title: 'Autocapture & heatmaps',
 
         settings: [
             {
                 id: 'autocapture',
                 title: 'Autocapture',
                 component: <AutocaptureSettings />,
             },
+            {
+                id: 'heatmaps',
+                title: 'Heatmaps',
+                component: <HeatmapsSettings />,
+                flag: 'TOOLBAR_HEATMAPS',
+            },
             {
                 id: 'exception-autocapture',
                 title: 'Exception autocapture',

diff --git a/...nes/settings/organization/VerifiedDomains/__snapshots__/verifiedDomainsLogic.test.ts.snap b/...nes/settings/organization/VerifiedDomains/__snapshots__/verifiedDomainsLogic.test.ts.snap
@@ -58,6 +58,7 @@ exports[`verifiedDomainsLogic values has proper defaults 1`] = `
         "effective_membership_level": 8,
         "groups_on_events_querying_enabled": true,
         "has_group_types": true,
+        "heatmaps_opt_in": true,
         "id": 997,
         "ingested_event": true,
         "is_demo": false,

diff --git a/frontend/src/scenes/settings/project/HeatmapsSettings.tsx b/frontend/src/scenes/settings/project/HeatmapsSettings.tsx
@@ -0,0 +1,40 @@
+import { LemonSwitch } from '@posthog/lemon-ui'
+import { useActions, useValues } from 'kea'
+import { eventUsageLogic } from 'lib/utils/eventUsageLogic'
+import { teamLogic } from 'scenes/teamLogic'
+import { userLogic } from 'scenes/userLogic'
+
+export function HeatmapsSettings(): JSX.Element {
+    const { userLoading } = useValues(userLogic)
+    const { currentTeam } = useValues(teamLogic)
+    const { updateCurrentTeam } = useActions(teamLogic)
+    const { reportHeatmapsToggled } = useActions(eventUsageLogic)
+
+    return (
+        <>
+            <p>
+                If you use our JavaScript libraries, we can capture general clicks, mouse movements, and scrolling to
+                create heatmaps. No additional events are created, and you can disable this at any time.
+                <br />
+                Whereas Autocapture creates events whenever it can uniquely identify an interacted element, heatmaps are
+                generated based on overall mouse or touch positions and are useful for understanding more general user
+                behavior across your site.
+            </p>
+            <div className="space-y-2">
+                <LemonSwitch
+                    id="posthog-heatmaps-switch"
+                    onChange={(checked) => {
+                        updateCurrentTeam({
+                            heatmaps_opt_in: checked,
+                        })
+                        reportHeatmapsToggled(checked)
+                    }}
+                    checked={!!currentTeam?.heatmaps_opt_in}
+                    disabled={userLoading}
+                    label="Enable heatmaps for web"
+                    bordered
+                />
+            </div>
+        </>
+    )
+}
diff --git a/frontend/src/scenes/settings/types.ts b/frontend/src/scenes/settings/types.ts
@@ -75,6 +75,7 @@ export type SettingId =
     | 'optout'
     | 'theme'
     | 'replay-ai-config'
+    | 'heatmaps'
 
 export type Setting = {
     id: SettingId

diff --git a/frontend/src/scenes/teamActivityDescriber.tsx b/frontend/src/scenes/teamActivityDescriber.tsx
@@ -150,7 +150,10 @@ const teamActionsMapping: Record<
     autocapture_exceptions_errors_to_ignore: () => null,
     autocapture_exceptions_opt_in: () => null,
     autocapture_opt_out(change: ActivityChange | undefined): ChangeMapping | null {
-        return { description: [<>{change?.after ? 'enabled' : 'disabled'} autocapture</>] }
+        return { description: [<>{change?.after ? 'opted in to' : 'opted out of'} autocapture</>] }
+    },
+    heatmaps_opt_in(change: ActivityChange | undefined): ChangeMapping | null {
+        return { description: [<>{change?.after ? 'enabled' : 'disabled'} heatmaps</>] }
     },
     // and.... many more
     name(change: ActivityChange | undefined): ChangeMapping | null {

diff --git a/frontend/src/types.ts b/frontend/src/types.ts
@@ -443,6 +443,7 @@ export interface TeamType extends TeamBasicType {
     session_replay_config: { record_canvas?: boolean; ai_config?: SessionRecordingAIConfig } | undefined | null
     autocapture_exceptions_opt_in: boolean
     surveys_opt_in?: boolean
+    heatmaps_opt_in?: boolean
     autocapture_exceptions_errors_to_ignore: string[]
     test_account_filters: AnyPropertyFilter[]
     test_account_filters_default_checked: boolean

diff --git a/latest_migrations.manifest b/latest_migrations.manifest
@@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name
 ee: 0016_rolemembership_organization_member
 otp_static: 0002_throttling
 otp_totp: 0002_auto_20190420_0723
-posthog: 0404_remove_propertydefinition_property_type_is_valid_and_more
+posthog: 0405_team_heatmaps_opt_in
 sessions: 0001_initial
 social_django: 0010_uid_db_index
 two_factor: 0007_auto_20201201_1019
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/posthog/api/app_metrics.py b/posthog/api/app_metrics.py
@@ -2,7 +2,7 @@
 import uuid
 from typing import Any
 
-from django.db.models import Sum
+from django.db.models import Q, Sum
 from django.db.models.functions import Coalesce, TruncDay
 from rest_framework import mixins, request, response, viewsets
 from rest_framework.decorators import action
@@ -82,13 +82,14 @@ def get_batch_export_runs_app_metrics_queryset(self, batch_export_id: str):
         ```
         select
             date_trunc('day', last_updated_at) as dates,
-            sum(coalesce(records_completed, 0)) as successes,
-            sum(coalesce(records_total_count, 0)) - sum(coalesce(records_completed, 0)) as failures
+            sum(case when status = 'Completed' then coalesce(records_total_count, 0) else 0) as successes,
+            sum(case when status != 'Completed' then coalesce(records_total_count, 0) else 0) as failures
         from
             posthog_batchexportrun
         where
             batch_export_id = :batch_export_id
             and last_updated_at between :date_from and :date_to
+            and status != 'Running'
         group by
             date_trunc('day', last_updated_at)
         order by
@@ -117,8 +118,8 @@ def get_batch_export_runs_app_metrics_queryset(self, batch_export_id: str):
             .annotate(dates=TruncDay("last_updated_at"))
             .values("dates")
             .annotate(
-                successes=Sum(Coalesce("records_completed", 0)),
-                failures=Sum(Coalesce("records_total_count", 0)) - Sum(Coalesce("records_completed", 0)),
+                successes=Sum(Coalesce("records_total_count", 0), filter=Q(status=BatchExportRun.Status.COMPLETED)),
+                failures=Sum(Coalesce("records_total_count", 0), filter=~Q(status=BatchExportRun.Status.COMPLETED)),
             )
             .order_by("dates")
             .all()