feat: session summaries (#19773)

A quick spike of summarizing sessions to seek feedback from everyone

it's relatively slow and costs money so I've wrapped it in a flag and am only summarizing one recording at a time

frontend/src/lib/api.ts

@@ -1537,6 +1537,12 @@ const api = {
             return await new ApiRequest().recording(recordingId).withAction('persist').create()
         },
 
+        async summarize(
+            recordingId: SessionRecordingType['id']
+        ): Promise<{ content: string; ai_result: Record<string, any> }> {
+            return await new ApiRequest().recording(recordingId).withAction('summarize').create()
+        },
+
         async delete(recordingId: SessionRecordingType['id']): Promise<{ success: boolean }> {
             return await new ApiRequest().recording(recordingId).delete()
         },

frontend/src/lib/constants.tsx

@@ -194,6 +194,7 @@ export const FEATURE_FLAGS = {
     REDIRECT_WEB_PRODUCT_ANALYTICS_ONBOARDING: 'redirect-web-product-analytics-onboarding', // owner: @biancayang
     RECRUIT_ANDROID_MOBILE_BETA_TESTERS: 'recruit-android-mobile-beta-testers', // owner: #team-replay
     SIDEPANEL_STATUS: 'sidepanel-status', // owner: @benjackwhite
+    AI_SESSION_SUMMARY: 'ai-session-summary', // owner: #team-replay
 } as const
 export type FeatureFlagKey = (typeof FEATURE_FLAGS)[keyof typeof FEATURE_FLAGS]
 

frontend/src/scenes/session-recordings/playlist/SessionRecordingPreview.tsx

@@ -1,12 +1,17 @@
 import clsx from 'clsx'
 import { useValues } from 'kea'
+import { FlaggedFeature } from 'lib/components/FlaggedFeature'
 import { PropertyIcon } from 'lib/components/PropertyIcon'
 import { TZLabel } from 'lib/components/TZLabel'
-import { IconAutocapture, IconKeyboard, IconPinFilled, IconSchedule } from 'lib/lemon-ui/icons'
+import { FEATURE_FLAGS } from 'lib/constants'
+import { IconAutoAwesome, IconAutocapture, IconKeyboard, IconPinFilled, IconSchedule } from 'lib/lemon-ui/icons'
+import { LemonButton } from 'lib/lemon-ui/LemonButton'
 import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton'
+import { Popover } from 'lib/lemon-ui/Popover'
+import { Spinner } from 'lib/lemon-ui/Spinner'
 import { Tooltip } from 'lib/lemon-ui/Tooltip'
 import { colonDelimitedDuration } from 'lib/utils'
-import { Fragment } from 'react'
+import { Fragment, useState } from 'react'
 import { DraggableToNotebook } from 'scenes/notebooks/AddToNotebook/DraggableToNotebook'
 import { asDisplay } from 'scenes/persons/person-utils'
 import { playerSettingsLogic } from 'scenes/session-recordings/player/playerSettingsLogic'
@@ -22,6 +27,8 @@ export interface SessionRecordingPreviewProps {
     isActive?: boolean
     onClick?: () => void
     pinned?: boolean
+    summariseFn?: (recording: SessionRecordingType) => void
+    sessionSummaryLoading?: boolean
 }
 
 function RecordingDuration({
@@ -228,18 +235,61 @@ export function SessionRecordingPreview({
     onClick,
     onPropertyClick,
     pinned,
+    summariseFn,
+    sessionSummaryLoading,
 }: SessionRecordingPreviewProps): JSX.Element {
     const { durationTypeToShow } = useValues(playerSettingsLogic)
 
     const iconClassnames = clsx('SessionRecordingPreview__property-icon text-base text-muted-alt')
 
+    const [summaryPopoverIsVisible, setSummaryPopoverIsVisible] = useState<boolean>(false)
+
+    const [summaryButtonIsVisible, setSummaryButtonIsVisible] = useState<boolean>(false)
+
     return (
         <DraggableToNotebook href={urls.replaySingle(recording.id)}>
             <div
                 key={recording.id}
                 className={clsx('SessionRecordingPreview', isActive && 'SessionRecordingPreview--active')}
                 onClick={() => onClick?.()}
+                onMouseEnter={() => setSummaryButtonIsVisible(true)}
+                onMouseLeave={() => setSummaryButtonIsVisible(false)}
             >
+                <FlaggedFeature flag={FEATURE_FLAGS.AI_SESSION_SUMMARY} match={true}>
+                    {summariseFn && (
+                        <Popover
+                            showArrow={true}
+                            visible={summaryPopoverIsVisible && summaryButtonIsVisible}
+                            placement="right"
+                            onClickOutside={() => setSummaryPopoverIsVisible(false)}
+                            overlay={
+                                sessionSummaryLoading ? (
+                                    <Spinner />
+                                ) : (
+                                    <div className="text-xl max-w-auto lg:max-w-3/5">{recording.summary}</div>
+                                )
+                            }
+                        >
+                            <LemonButton
+                                size="small"
+                                type="primary"
+                                className={clsx(
+                                    summaryButtonIsVisible ? 'block' : 'hidden',
+                                    'absolute right-px top-px'
+                                )}
+                                icon={<IconAutoAwesome />}
+                                onClick={(e) => {
+                                    e.preventDefault()
+                                    e.stopPropagation()
+                                    setSummaryPopoverIsVisible(!summaryPopoverIsVisible)
+                                    if (!recording.summary) {
+                                        summariseFn(recording)
+                                    }
+                                }}
+                            />
+                        </Popover>
+                    )}
+                </FlaggedFeature>
                 <div className="grow overflow-hidden space-y-px">
                     <div className="flex items-center justify-between gap-2">
                         <div className="flex items-center gap-1 shrink overflow-hidden">

frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx

@@ -92,6 +92,8 @@ function RecordingsLists(): JSX.Element {
         logicProps,
         showOtherRecordings,
         recordingsCount,
+        sessionSummaryLoading,
+        sessionBeingSummarized,
     } = useValues(sessionRecordingsPlaylistLogic)
     const {
         setSelectedRecordingId,
@@ -102,6 +104,7 @@ function RecordingsLists(): JSX.Element {
         resetFilters,
         setShowAdvancedFilters,
         toggleShowOtherRecordings,
+        summarizeSession,
     } = useActions(sessionRecordingsPlaylistLogic)
 
     const onRecordingClick = (recording: SessionRecordingType): void => {
@@ -112,6 +115,10 @@ function RecordingsLists(): JSX.Element {
         setFilters(defaultPageviewPropertyEntityFilter(filters, property, value))
     }
 
+    const onSummarizeClick = (recording: SessionRecordingType): void => {
+        summarizeSession(recording.id)
+    }
+
     const lastScrollPositionRef = useRef(0)
     const contentRef = useRef<HTMLDivElement | null>(null)
 
@@ -248,6 +255,10 @@ function RecordingsLists(): JSX.Element {
                                               onPropertyClick={onPropertyClick}
                                               isActive={activeSessionRecordingId === rec.id}
                                               pinned={false}
+                                              summariseFn={onSummarizeClick}
+                                              sessionSummaryLoading={
+                                                  sessionSummaryLoading && sessionBeingSummarized === rec.id
+                                              }
                                           />
                                       </div>
                                   ))

frontend/src/scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic.ts

@@ -205,6 +205,11 @@ export interface SessionRecordingPlaylistLogicProps {
     onPinnedChange?: (recording: SessionRecordingType, pinned: boolean) => void
 }
 
+export interface SessionSummaryResponse {
+    id: SessionRecordingType['id']
+    content: string
+}
+
 export const sessionRecordingsPlaylistLogic = kea<sessionRecordingsPlaylistLogicType>([
     path((key) => ['scenes', 'session-recordings', 'playlist', 'sessionRecordingsPlaylistLogic', key]),
     props({} as SessionRecordingPlaylistLogicProps),
@@ -239,6 +244,7 @@ export const sessionRecordingsPlaylistLogic = kea<sessionRecordingsPlaylistLogic
         loadPinnedRecordings: true,
         loadSessionRecordings: (direction?: 'newer' | 'older') => ({ direction }),
         maybeLoadSessionRecordings: (direction?: 'newer' | 'older') => ({ direction }),
+        summarizeSession: (id: SessionRecordingType['id']) => ({ id }),
         loadNext: true,
         loadPrev: true,
         toggleShowOtherRecordings: (show?: boolean) => ({ show }),
@@ -255,6 +261,15 @@ export const sessionRecordingsPlaylistLogic = kea<sessionRecordingsPlaylistLogic
     }),
 
     loaders(({ props, values, actions }) => ({
+        sessionSummary: {
+            summarizeSession: async ({ id }): Promise<SessionSummaryResponse | null> => {
+                if (!id) {
+                    return null
+                }
+                const response = await api.recordings.summarize(id)
+                return { content: response.content, id: id }
+            },
+        },
         eventsHaveSessionId: [
             {} as Record<string, boolean>,
             {
@@ -342,6 +357,13 @@ export const sessionRecordingsPlaylistLogic = kea<sessionRecordingsPlaylistLogic
         ],
     })),
     reducers(({ props }) => ({
+        sessionBeingSummarized: [
+            null as null | SessionRecordingType['id'],
+            {
+                summarizeSession: (_, { id }) => id,
+                sessionSummarySuccess: () => null,
+            },
+        ],
         // If we initialise with pinned recordings then we don't show others by default
         // but if we go down to 0 pinned recordings then we show others
         showOtherRecordings: [
@@ -428,6 +450,7 @@ export const sessionRecordingsPlaylistLogic = kea<sessionRecordingsPlaylistLogic
 
                     return mergedResults
                 },
+
                 setSelectedRecordingId: (state, { id }) =>
                     state.map((s) => {
                         if (s.id === id) {
@@ -439,6 +462,21 @@ export const sessionRecordingsPlaylistLogic = kea<sessionRecordingsPlaylistLogic
                             return { ...s }
                         }
                     }),
+
+                summarizeSessionSuccess: (state, { sessionSummary }) => {
+                    return sessionSummary
+                        ? state.map((s) => {
+                              if (s.id === sessionSummary.id) {
+                                  return {
+                                      ...s,
+                                      summary: sessionSummary.content,
+                                  }
+                              } else {
+                                  return s
+                              }
+                          })
+                        : state
+                },
             },
         ],
         selectedRecordingId: [

frontend/src/types.ts

@@ -1100,6 +1100,7 @@ export interface SessionRecordingType {
     console_error_count?: number
     /** Where this recording information was loaded from  */
     storage?: 'object_storage_lts' | 'object_storage'
+    summary?: string
 }
 
 export interface SessionRecordingPropertiesType {

posthog/session_recordings/queries/session_replay_events.py

@@ -6,8 +6,10 @@
 from posthog.clickhouse.client import sync_execute
 from posthog.cloud_utils import is_cloud
 from posthog.constants import AvailableFeature
+
 from posthog.models.instance_setting import get_instance_setting
 from posthog.models.team import Team
+
 from posthog.session_recordings.models.metadata import (
     RecordingMetadata,
 )
@@ -102,6 +104,40 @@ def get_metadata(
             console_error_count=replay[11],
         )
 
+    def get_events(
+        self, session_id: str, team: Team, metadata: RecordingMetadata, events_to_ignore: List[str] | None
+    ) -> Tuple[List | None, List | None]:
+        from posthog.schema import HogQLQuery, HogQLQueryResponse
+        from posthog.hogql_queries.hogql_query_runner import HogQLQueryRunner
+
+        q = """
+            select event, timestamp, elements_chain, properties.$window_id, properties.$current_url, properties.$event_type
+            from events
+            where timestamp >= {start_time} and timestamp <= {end_time}
+            and $session_id = {session_id}
+            """
+        if events_to_ignore:
+            q += " and event not in {events_to_ignore}"
+
+        q += " order by timestamp asc"
+
+        hq = HogQLQuery(
+            query=q,
+            values={
+                "start_time": metadata["start_time"],
+                "end_time": metadata["end_time"],
+                "session_id": session_id,
+                "events_to_ignore": events_to_ignore,
+            },
+        )
+
+        result: HogQLQueryResponse = HogQLQueryRunner(
+            team=team,
+            query=hq,
+        ).calculate()
+
+        return result.columns, result.results
+
 
 def ttl_days(team: Team) -> int:
     ttl_days = (get_instance_setting("RECORDINGS_TTL_WEEKS") or 3) * 7