feat: canvas replay

frontend/src/@types/rrweb.d.ts

@@ -0,0 +1 @@
+declare module 'rrweb/es/rrweb/packages/rrweb/src/replay/canvas'

frontend/src/lib/api.mock.ts

@@ -73,6 +73,7 @@ export const MOCK_DEFAULT_TEAM: TeamType = {
     session_recording_minimum_duration_milliseconds: null,
     session_recording_linked_flag: null,
     session_recording_network_payload_capture_config: null,
+    session_replay_config: null,
     capture_console_log_opt_in: true,
     capture_performance_opt_in: true,
     autocapture_exceptions_opt_in: false,

frontend/src/lib/constants.tsx

@@ -192,6 +192,7 @@ export const FEATURE_FLAGS = {
     SESSION_REPLAY_IOS: 'session-replay-ios', // owner: #team-replay
     YEAR_IN_HOG: 'year-in-hog', // owner: #team-replay
     SESSION_REPLAY_EXPORT_MOBILE_DATA: 'session-replay-export-mobile-data', // owner: #team-replay
+    SESSION_REPLAY_CANVAS: 'session-replay-canvas', // owner: #team-replay
     DISCUSSIONS: 'discussions', // owner: #team-replay
     REDIRECT_WEB_PRODUCT_ANALYTICS_ONBOARDING: 'redirect-web-product-analytics-onboarding', // owner: @biancayang
     RECRUIT_ANDROID_MOBILE_BETA_TESTERS: 'recruit-android-mobile-beta-testers', // owner: #team-replay

frontend/src/lib/utils.tsx

@@ -1714,11 +1714,20 @@ export const base64Encode = (str: string): string => {
 }
 
 export const base64Decode = (encodedString: string): string => {
+    const data = base64ToUint8Array(encodedString)
+    return new TextDecoder().decode(data)
+}
+
+export const base64ArrayBuffer = (encodedString: string): ArrayBuffer => {
+    const data = base64ToUint8Array(encodedString)
+    return data.buffer
+}
+
+export const base64ToUint8Array = (encodedString: string): Uint8Array => {
     const binString = atob(encodedString)
     const data = new Uint8Array(binString.length)
     for (let i = 0; i < binString.length; i++) {
         data[i] = binString.charCodeAt(i)
     }
-
-    return new TextDecoder().decode(data)
+    return data
 }

frontend/src/scenes/session-recordings/player/rrweb/canvas/canvas-plugin.ts

@@ -0,0 +1,94 @@
+import { CanvasArg, canvasMutationData, canvasMutationParam, eventWithTime } from '@rrweb/types'
+import { EventType, IncrementalSource, Replayer } from 'rrweb'
+import { canvasMutation } from 'rrweb/es/rrweb/packages/rrweb/src/replay/canvas'
+import { ReplayPlugin } from 'rrweb/typings/types'
+
+import { deserializeCanvasArg } from './deserialize-canvas-args'
+
+export const CanvasReplayerPlugin = (events: eventWithTime[]): ReplayPlugin => {
+    const canvases = new Map<number, HTMLCanvasElement>([])
+    const containers = new Map<number, HTMLImageElement>([])
+    const imageMap = new Map<eventWithTime | string, HTMLImageElement>()
+    const canvasEventMap = new Map<eventWithTime | string, canvasMutationParam>()
+
+    const deserializeAndPreloadCanvasEvents = async (data: canvasMutationData, event: eventWithTime): Promise<void> => {
+        if (!canvasEventMap.has(event)) {
+            const status = { isUnchanged: true }
+
+            if ('commands' in data) {
+                const commands = await Promise.all(
+                    data.commands.map(async (c) => {
+                        const args = await Promise.all(
+                            (c.args as CanvasArg[]).map(deserializeCanvasArg(imageMap, null, status))
+                        )
+                        return { ...c, args }
+                    })
+                )
+                if (status.isUnchanged === false) {
+                    canvasEventMap.set(event, { ...data, commands })
+                }
+            } else {
+                const args = await Promise.all(
+                    (data.args as CanvasArg[]).map(deserializeCanvasArg(imageMap, null, status))
+                )
+                if (status.isUnchanged === false) {
+                    canvasEventMap.set(event, { ...data, args })
+                }
+            }
+        }
+    }
+
+    const cloneCanvas = (id: number, node: HTMLCanvasElement): HTMLCanvasElement => {
+        const cloneNode = node.cloneNode() as HTMLCanvasElement
+        canvases.set(id, cloneNode)
+        document.adoptNode(cloneNode)
+        return cloneNode
+    }
+
+    const promises: Promise<any>[] = []
+    for (const event of events) {
+        if (event.type === EventType.IncrementalSnapshot && event.data.source === IncrementalSource.CanvasMutation) {
+            promises.push(deserializeAndPreloadCanvasEvents(event.data, event))
+        }
+    }
+
+    return {
+        onBuild: (node, { id }) => {
+            if (!node) {
+                return
+            }
+
+            if (node.nodeName === 'CANVAS' && node.nodeType === 1) {
+                const el = containers.get(id) || document.createElement('img')
+                ;(node as HTMLCanvasElement).appendChild(el)
+                containers.set(id, el)
+            }
+        },
+
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        handler: async (e: eventWithTime, _isSync: boolean, { replayer }: { replayer: Replayer }) => {
+            if (e.type === EventType.IncrementalSnapshot && e.data.source === IncrementalSource.CanvasMutation) {
+                const source = replayer.getMirror().getNode(e.data.id)
+                const target =
+                    canvases.get(e.data.id) || (source && cloneCanvas(e.data.id, source as HTMLCanvasElement))
+
+                if (!target) {
+                    return
+                }
+
+                await canvasMutation({
+                    event: e,
+                    mutation: e.data,
+                    target: target,
+                    imageMap,
+                    canvasEventMap,
+                })
+
+                const img = containers.get(e.data.id)
+                if (img) {
+                    img.src = target.toDataURL()
+                }
+            }
+        },
+    } as ReplayPlugin
+}

frontend/src/scenes/session-recordings/player/rrweb/canvas/deserialize-canvas-args.ts

@@ -0,0 +1,68 @@
+import { CanvasArg } from '@rrweb/types'
+import { base64ArrayBuffer } from 'lib/utils'
+import { Replayer } from 'rrweb'
+
+type GLVarMap = Map<string, any[]>
+type CanvasContexts = CanvasRenderingContext2D | WebGLRenderingContext | WebGL2RenderingContext
+const webGLVarMap: Map<CanvasContexts, GLVarMap> = new Map()
+
+const variableListFor = (ctx: CanvasContexts, ctor: string): any[] => {
+    let contextMap = webGLVarMap.get(ctx)
+    if (!contextMap) {
+        contextMap = new Map()
+        webGLVarMap.set(ctx, contextMap)
+    }
+    if (!contextMap.has(ctor)) {
+        contextMap.set(ctor, [])
+    }
+
+    return contextMap.get(ctor) as any[]
+}
+
+export const deserializeCanvasArg = (
+    imageMap: Replayer['imageMap'],
+    ctx: CanvasContexts | null,
+    preload?: {
+        isUnchanged: boolean
+    }
+): ((arg: CanvasArg) => Promise<any>) => {
+    return async (arg: CanvasArg): Promise<any> => {
+        if (arg && typeof arg === 'object' && 'rr_type' in arg) {
+            if (preload) {
+                preload.isUnchanged = false
+            }
+            if (arg.rr_type === 'ImageBitmap' && 'args' in arg) {
+                const args = await deserializeCanvasArg(imageMap, ctx, preload)(arg.args)
+                // eslint-disable-next-line prefer-spread
+                return await createImageBitmap.apply(null, args)
+            }
+            if ('index' in arg) {
+                if (preload || ctx === null) {
+                    return arg
+                }
+                const { rr_type: name, index } = arg
+                return variableListFor(ctx, name)[index]
+            }
+            if ('args' in arg) {
+                return arg
+            }
+            if ('base64' in arg) {
+                return base64ArrayBuffer(arg.base64)
+            }
+            if ('src' in arg) {
+                return arg
+            }
+            if ('data' in arg && arg.rr_type === 'Blob') {
+                const blobContents = await Promise.all(arg.data.map(deserializeCanvasArg(imageMap, ctx, preload)))
+                const blob = new Blob(blobContents, {
+                    type: arg.type,
+                })
+                return blob
+            }
+        } else if (Array.isArray(arg)) {
+            const result = await Promise.all(arg.map(deserializeCanvasArg(imageMap, ctx, preload)))
+            return result
+        }
+        return arg
+    }
+}

frontend/src/scenes/session-recordings/player/sessionRecordingPlayerLogic.ts

@@ -44,6 +44,7 @@ import { createExportedSessionRecording } from '../file-playback/sessionRecordin
 import type { sessionRecordingsPlaylistLogicType } from '../playlist/sessionRecordingsPlaylistLogicType'
 import { playerSettingsLogic } from './playerSettingsLogic'
 import { COMMON_REPLAYER_CONFIG, CorsPlugin } from './rrweb'
+import { CanvasReplayerPlugin } from './rrweb/canvas/canvas-plugin'
 import type { sessionRecordingPlayerLogicType } from './sessionRecordingPlayerLogicType'
 import { deleteRecording } from './utils/playerUtils'
 import { SessionRecordingPlayerExplorerProps } from './view-explorer/SessionRecordingPlayerExplorer'
@@ -526,6 +527,10 @@ export const sessionRecordingPlayerLogic = kea<sessionRecordingPlayerLogicType>(
                 plugins.push(CorsPlugin)
             }
 
+            if (values.featureFlags[FEATURE_FLAGS.SESSION_REPLAY_CANVAS]) {
+                plugins.push(CanvasReplayerPlugin(values.sessionPlayerData.snapshotsByWindowId[windowId]))
+            }
+
             cache.debug?.('tryInitReplayer', {
                 windowId,
                 rootFrame: values.rootFrame,

frontend/src/scenes/settings/organization/VerifiedDomains/__snapshots__/verifiedDomainsLogic.test.ts.snap

@@ -79,6 +79,7 @@ exports[`verifiedDomainsLogic values has proper defaults 1`] = `
         "session_recording_network_payload_capture_config": null,
         "session_recording_opt_in": true,
         "session_recording_sample_rate": "1.0",
+        "session_replay_config": null,
         "slack_incoming_webhook": "",
         "test_account_filters": [
           {

frontend/src/scenes/settings/project/SessionRecordingSettings.tsx

@@ -1,10 +1,11 @@
-import { LemonButton, LemonSelect, LemonSwitch, Link } from '@posthog/lemon-ui'
+import { LemonButton, LemonSelect, LemonSwitch, LemonTag, Link } from '@posthog/lemon-ui'
 import { useActions, useValues } from 'kea'
 import { AuthorizedUrlList } from 'lib/components/AuthorizedUrlList/AuthorizedUrlList'
 import { AuthorizedUrlListType } from 'lib/components/AuthorizedUrlList/authorizedUrlListLogic'
 import { FlaggedFeature } from 'lib/components/FlaggedFeature'
 import { FlagSelector } from 'lib/components/FlagSelector'
 import { FEATURE_FLAGS, SESSION_REPLAY_MINIMUM_DURATION_OPTIONS } from 'lib/constants'
+import { useFeatureFlag } from 'lib/hooks/useFeatureFlag'
 import { IconCancel } from 'lib/lemon-ui/icons'
 import { LemonLabel } from 'lib/lemon-ui/LemonLabel/LemonLabel'
 import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
@@ -16,8 +17,8 @@ import { AvailableFeature } from '~/types'
 
 export function ReplayGeneral(): JSX.Element {
     const { updateCurrentTeam } = useActions(teamLogic)
-
     const { currentTeam } = useValues(teamLogic)
+    const hasCanvasRecording = useFeatureFlag('SESSION_REPLAY_CANVAS')
 
     return (
         <div className="space-y-4">
@@ -73,6 +74,42 @@ export function ReplayGeneral(): JSX.Element {
                     logs will be shown in the recording player to help you debug any issues.
                 </p>
             </div>
+            {hasCanvasRecording && (
+                <div className="space-y-2">
+                    <LemonSwitch
+                        data-attr="opt-in-capture-canvas-switch"
+                        onChange={(checked) => {
+                            updateCurrentTeam({
+                                session_replay_config: {
+                                    ...currentTeam?.session_replay_config,
+                                    record_canvas: checked,
+                                },
+                            })
+                        }}
+                        label={
+                            <div className="space-x-1">
+                                <LemonTag type="success">New</LemonTag>
+                                <LemonLabel>Capture canvas elements</LemonLabel>
+                            </div>
+                        }
+                        bordered
+                        checked={
+                            currentTeam?.session_replay_config
+                                ? !!currentTeam?.session_replay_config?.record_canvas
+                                : false
+                        }
+                    />
+                    <p>
+                        This setting controls if browser canvas elements will be captured as part of recordings.{' '}
+                        <b>
+                            <i>
+                                There is no way to mask canvas elements right now so please make sure they are free of
+                                PII.
+                            </i>
+                        </b>
+                    </p>
+                </div>
+            )}
             <div className="space-y-2">
                 <NetworkCaptureSettings />
             </div>

frontend/src/types.ts

@@ -360,6 +360,7 @@ export interface TeamType extends TeamBasicType {
         | { recordHeaders?: boolean; recordBody?: boolean }
         | undefined
         | null
+    session_replay_config: { record_canvas?: boolean } | undefined | null
     autocapture_exceptions_opt_in: boolean
     surveys_opt_in?: boolean
     autocapture_exceptions_errors_to_ignore: string[]

jest.config.ts

@@ -103,6 +103,7 @@ const config: Config = {
         '^scenes/(.*)$': '<rootDir>/frontend/src/scenes/$1',
         '^antd/es/(.*)$': 'antd/lib/$1',
         '^react-virtualized/dist/es/(.*)$': 'react-virtualized/dist/commonjs/$1',
+        '^rrweb/es/rrweb': 'rrweb/dist/rrweb.min.js',
         d3: '<rootDir>/node_modules/d3/dist/d3.min.js',
         '^d3-(.*)$': `d3-$1/dist/d3-$1`,
     },

latest_migrations.manifest

@@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name
 ee: 0015_add_verified_properties
 otp_static: 0002_throttling
 otp_totp: 0002_auto_20190420_0723
-posthog: 0385_exception_autocapture_off_for_all
+posthog: 0386_add_session_replay_config_to_team
 sessions: 0001_initial
 social_django: 0010_uid_db_index
 two_factor: 0007_auto_20201201_1019

posthog/api/decide.py

@@ -257,7 +257,7 @@ def get_decide(request: HttpRequest):
                 if isinstance(linked_flag, Dict):
                     linked_flag = linked_flag.get("key")
 
-                response["sessionRecording"] = {
+                session_recording_response = {
                     "endpoint": "/s/",
                     "consoleLogRecordingEnabled": capture_console_logs,
                     "recorderVersion": "v2",
@@ -267,6 +267,19 @@ def get_decide(request: HttpRequest):
                     "networkPayloadCapture": team.session_recording_network_payload_capture_config or None,
                 }
 
+                if isinstance(team.session_replay_config, Dict):
+                    record_canvas = team.session_replay_config["record_canvas"] or False
+                    session_recording_response.update(
+                        {
+                            "recordCanvas": record_canvas,
+                            # hard coded during beta while we decide on sensible values
+                            "canvasFps": 4 if record_canvas else None,
+                            "canvasQuality": "0.6" if record_canvas else None,
+                        }
+                    )
+
+                response["sessionRecording"] = session_recording_response
+
             response["surveys"] = True if team.surveys_opt_in else False
 
             site_apps = []

posthog/api/team.py

@@ -102,6 +102,7 @@ class Meta:
             "session_recording_minimum_duration_milliseconds",
             "session_recording_linked_flag",
             "session_recording_network_payload_capture_config",
+            "session_replay_config",
             "recording_domains",
             "inject_web_apps",
             "surveys_opt_in",
@@ -146,6 +147,7 @@ class Meta:
             "session_recording_minimum_duration_milliseconds",
             "session_recording_linked_flag",
             "session_recording_network_payload_capture_config",
+            "session_replay_config",
             "effective_membership_level",
             "access_control",
             "week_start_day",
@@ -208,6 +210,18 @@ def validate_session_recording_network_payload_capture_config(self, value) -> Di
 
         return value
 
+    def validate_session_replay_config(self, value) -> Dict | None:
+        if value is None:
+            return None
+
+        if not isinstance(value, Dict):
+            raise exceptions.ValidationError("Must provide a dictionary or None.")
+
+        if not all(key in ["record_canvas"] for key in value.keys()):
+            raise exceptions.ValidationError("Must provide a dictionary with only 'record_canvas' key.")
+
+        return value
+
     def validate(self, attrs: Any) -> Any:
         if "primary_dashboard" in attrs and attrs["primary_dashboard"].team != self.instance:
             raise exceptions.PermissionDenied("Dashboard does not belong to this team.")